traject 3.8.3 → 3.9.1

data/test/solr_json_httpx_writer_test.rb

@@ -0,0 +1,428 @@
+require 'test_helper'
+require 'httpx'
+require 'traject/solr_json_httpx_writer'
+require 'thread'
+require 'json'
+require 'stringio'
+require 'logger'
+
+
+require "httpx/adapters/webmock"
+WebMock.enable! # not sure why we need this
+
+
+# Some basic tests, using a mocked http client so we can see what it did --
+# these tests do not run against a real solr server at present.
+describe "Traject::SolrJsonHttpxWriter" do
+  TEST_SOLR_URL = "http://example.com/solr"
+
+  #######
+  # utilities to help testing
+  #######
+
+  def context_with(hash)
+    Traject::Indexer::Context.new(:output_hash => hash)
+  end
+
+  def create_writer(settings = {})
+    settings = {
+      "solr.url" => TEST_SOLR_URL,
+      }.merge!(settings)
+
+    Traject::SolrJsonHttpxWriter.new(settings)
+  end
+
+  # strio = StringIO.new
+  # logger_to_strio(strio)
+  #
+  # Later check for strio.string for contents
+  def logger_to_strio(strio)
+    # Yell makes this hard, let's do it with an ordinary logger, think
+    # it's okay.
+    Logger.new(strio)
+  end
+
+  #########
+  # Actual tests
+  #########
+
+  before do
+    @writer = create_writer
+  end
+
+  it "defaults to 1 bg thread" do
+    assert_equal 1, @writer.thread_pool_size
+  end
+
+  it "adds a document" do
+    stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 200)
+
+    @writer.put context_with({"id" => "one", "key" => ["value1", "value2"]})
+    @writer.close
+
+    assert_requested :post, "#{TEST_SOLR_URL}/update/json" do |request|
+      posted_json = JSON.parse(request.body)
+      assert_equal [{"id" => "one", "key" => ["value1", "value2"]}], posted_json
+    end
+  end
+
+  it "adds more than a batch in batches" do
+    stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 200)
+
+    (Traject::SolrJsonHttpxWriter::DEFAULT_BATCH_SIZE + 1).times do |i|
+      doc = {"id" => "doc_#{i}", "key" => "value"}
+      @writer.put context_with(doc)
+    end
+    @writer.close
+
+    assert_requested(:post, "#{TEST_SOLR_URL}/update/json", times: 2)
+
+    # first batch with 100
+    assert_requested(:post, "#{TEST_SOLR_URL}/update/json") do |request|
+      posted_json = JSON.parse(request.body)
+      posted_json.length == Traject::SolrJsonHttpxWriter::DEFAULT_BATCH_SIZE
+    end
+
+    # second batch with just one
+    assert_requested(:post, "#{TEST_SOLR_URL}/update/json") do |request|
+      posted_json = JSON.parse(request.body)
+      posted_json.length == 1
+    end
+  end
+
+  it "retries batch as individual records on failure" do
+    # capture post bodies for easier testing
+    post_bodies = []
+    stub_request(:post, "#{TEST_SOLR_URL}/update/json").with { |request|
+      post_bodies << JSON.parse(request.body)
+      true
+    }.to_return(status: 500)
+
+    @writer = create_writer("solr_writer.batch_size" => 2, "solr_writer.max_skipped" => 10)
+
+    2.times do |i|
+      @writer.put context_with({"id" => "doc_#{i}", "key" => "value"})
+    end
+    @writer.close
+
+    # 1 batch, then 2 for re-trying each individually
+    assert_requested(:post, "#{TEST_SOLR_URL}/update/json", times: 3)
+
+    batch_update = post_bodies.first
+
+    assert_length 2, batch_update
+
+    individual_update1, individual_update2 = post_bodies[1], post_bodies[2]
+    assert_length 1, individual_update1
+    assert_length 1, individual_update2
+  end
+
+  it "includes Solr reported error in base error message" do
+    stub_request(:post, "#{TEST_SOLR_URL}/update/json").
+      to_return(
+        status: 400,
+        headers: { content_type: "application/json;charset=utf-8" },
+        body: { "responseHeader"=>{"status"=>400, "QTime"=>0},
+                "error"=>{
+                  "metadata"=>["error-class", "org.apache.solr.common.SolrException", "root-error-class", "org.apache.solr.common.SolrException"],
+                  "msg"=>"ERROR: this is a solr error",
+                  "code"=>400
+                }
+              }.to_json
+      )
+
+
+    @writer = create_writer("solr_writer.batch_size" => 1, "solr_writer.max_skipped" => 0)
+
+    error = assert_raises(Traject::SolrJsonHttpxWriter::MaxSkippedRecordsExceeded) {
+      @writer.put context_with({"id" => "doc_1", "key" => "value"})
+      @writer.close
+    }
+    assert_match(/ERROR: this is a solr error/, error.message)
+  end
+
+  it "can #flush" do
+    stub_request(:post, "#{TEST_SOLR_URL}/update/json")
+
+    2.times do |i|
+      doc = {"id" => "doc_#{i}", "key" => "value"}
+      @writer.put context_with(doc)
+    end
+
+    # Hasn't yet written
+    assert_not_requested(:post, "#{TEST_SOLR_URL}/update/json")
+
+    @writer.flush
+
+    # Has flushed to solr
+    assert_requested(:post, "#{TEST_SOLR_URL}/update/json")
+  end
+
+  # We check the header on an actual (mocked) request, rather than looking at
+  # the httpx client's configured options -- httpx 1.7 moved basic auth out of
+  # the client's default headers and into an option only applied at request time.
+  AUTH_TEST_URL = "http://example.com/solr/foo"
+
+  def write_one_doc(writer)
+    writer.put context_with({"id" => "one"})
+    writer.close
+  end
+
+  it "defaults to not setting basic authentication" do
+    stub_request(:post, "#{AUTH_TEST_URL}/update/json")
+
+    write_one_doc Traject::SolrJsonHttpxWriter.new("solr.url" => AUTH_TEST_URL)
+
+    assert_requested(:post, "#{AUTH_TEST_URL}/update/json") do |request|
+      request.headers["Authorization"].nil?
+    end
+  end
+
+  describe "HTTP basic auth" do
+    let(:expected_auth_header) { "Basic #{Base64.strict_encode64("foo:bar")}" }
+
+    it "supports basic authentication settings" do
+      stub_request(:post, "#{AUTH_TEST_URL}/update/json")
+
+      write_one_doc Traject::SolrJsonHttpxWriter.new(
+        "solr.url" => AUTH_TEST_URL,
+        "solr_writer.basic_auth_user" => "foo",
+        "solr_writer.basic_auth_password" => "bar",
+      )
+
+      assert_requested(:post, "#{AUTH_TEST_URL}/update/json",
+        headers: { "Authorization" => expected_auth_header })
+    end
+
+    it "supports basic auth from solr.url" do
+      stub_request(:post, "#{AUTH_TEST_URL}/update/json")
+
+      write_one_doc Traject::SolrJsonHttpxWriter.new("solr.url" => "http://foo:bar@example.com/solr/foo")
+
+      assert_requested(:post, "#{AUTH_TEST_URL}/update/json",
+        headers: { "Authorization" => expected_auth_header })
+    end
+
+    it "does not log basic auth from solr.url" do
+      string_io = StringIO.new
+      settings = {
+        "solr.url" => "http://secret_username:secret_password@example.com/solr/foo",
+        "logger"   => Logger.new(string_io)
+      }
+
+
+      writer = Traject::SolrJsonHttpxWriter.new(settings)
+
+      refute_includes string_io.string, "secret_username:secret_password"
+      assert_includes string_io.string, "(with HTTP basic auth)"
+    end
+  end
+
+  describe "commit" do
+    it "commits on close when set" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json")
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+
+      @writer = create_writer("solr.url" => TEST_SOLR_URL, "solr_writer.commit_on_close" => "true")
+      @writer.put context_with({"id" => "one", "key" => ["value1", "value2"]})
+      @writer.close
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+    end
+
+    it "commits on close with commit_solr_update_args" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json")
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+
+      @writer = create_writer(
+        "solr.url" => TEST_SOLR_URL,
+        "solr_writer.commit_on_close" => "true",
+        "solr_writer.commit_solr_update_args" => { softCommit: true }
+      )
+      @writer.put context_with({"id" => "one", "key" => ["value1", "value2"]})
+      @writer.close
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+    end
+
+    it "can manually send commit" do
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+
+      @writer = create_writer("solr.url" => TEST_SOLR_URL)
+      @writer.commit
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+    end
+
+    it "can manually send commit with specified args" do
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?commit=true&optimize=true&waitFlush=false")
+
+      @writer = create_writer("solr.url" => TEST_SOLR_URL, "solr_writer.commit_solr_update_args" => { softCommit: true })
+      @writer.commit(commit: true, optimize: true, waitFlush: false)
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?commit=true&optimize=true&waitFlush=false")
+    end
+
+    it "uses commit_solr_update_args settings by default" do
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+
+      @writer = create_writer(
+        "solr.url" => TEST_SOLR_URL,
+        "solr_writer.commit_solr_update_args" => { softCommit: true }
+      )
+      @writer.commit
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+    end
+
+    it "overrides commit_solr_update_args with method arg" do
+      stub_request(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+
+      @writer = create_writer(
+        "solr.url" => TEST_SOLR_URL,
+        "solr_writer.commit_solr_update_args" => { softCommit: true, foo: "bar" }
+      )
+      @writer.commit(commit: true)
+
+      assert_requested(:get, "#{TEST_SOLR_URL}/update/json?commit=true")
+    end
+  end
+
+  describe "solr_writer.solr_update_args" do
+    before do
+      @writer = create_writer("solr_writer.solr_update_args" => { softCommit: true } )
+    end
+
+    it "sends update args" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+
+      @writer.put context_with({"id" => "one", "key" => ["value1", "value2"]})
+      @writer.close
+
+      assert_requested(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+    end
+
+    it "sends update args with delete" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true")
+
+      @writer.delete("test-id")
+      @writer.close
+
+      assert_requested(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true", times: 1)
+    end
+
+    it "sends update args on individual-retry after batch failure" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true").to_return(status: 500)
+
+      @writer = create_writer(
+        "solr_writer.batch_size" => 2,
+        "solr_writer.max_skipped" => 10,
+        "solr_writer.solr_update_args" => { softCommit: true }
+      )
+
+      2.times do |i|
+        @writer.put context_with({"id" => "doc_#{i}", "key" => "value"})
+      end
+      @writer.close
+
+      # 1 batch, then 2 for re-trying each individually
+      assert_requested(:post, "#{TEST_SOLR_URL}/update/json?softCommit=true", times: 3)
+    end
+  end
+
+  describe "skipped records" do
+    it "skips and reports under max_skipped" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 500)
+
+      strio = StringIO.new
+      @writer = create_writer("solr_writer.max_skipped" => 10, "logger" => logger_to_strio(strio))
+
+      10.times do |i|
+        @writer.put context_with("id" => "doc_#{i}", "key" => "value")
+      end
+      @writer.close
+
+      assert_equal 10, @writer.skipped_record_count
+
+      logged = strio.string
+
+      10.times do |i|
+        assert_match(/ERROR.*Could not add record <output_id:doc_#{i}>: Solr error response: 500/, logged)
+      end
+    end
+
+    it "raises when skipped more than max_skipped" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 500)
+
+      @writer = create_writer("solr_writer.max_skipped" => 5)
+
+      e = assert_raises(RuntimeError) do
+        6.times do |i|
+          @writer.put context_with("id" => "doc_#{i}", "key" => "value")
+        end
+        @writer.close
+      end
+
+      assert_includes e.message, "Exceeded maximum number of skipped records"
+    end
+
+    it "raises on one skipped record when max_skipped is 0" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 500)
+
+      @writer = create_writer("solr_writer.max_skipped" => 0)
+
+      _e = assert_raises(RuntimeError) do
+        @writer.put context_with("id" => "doc_1", "key" => "value")
+        @writer.close
+      end
+    end
+
+
+    it "when catching additional skip errors, raise RuntimeError" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_raise(ArgumentError.new('bad stuff'))
+
+      strio = StringIO.new
+      @writer = create_writer(
+        "solr_writer.max_skipped" => 0,
+        "logger" => logger_to_strio(strio),
+        "solr_writer.skippable_exceptions" => [ArgumentError]
+      )
+
+       _e = assert_raises(Traject::SolrJsonHttpxWriter::MaxSkippedRecordsExceeded) do
+        @writer.put context_with("id" => "doc_1", "key" => "value")
+        @writer.close
+      end
+       logged = strio.string
+      assert_includes logged, 'ArgumentError: bad stuff'
+    end
+  end
+
+  describe "#delete" do
+    it "deletes" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json")
+
+      id = "123456"
+      @writer.delete(id)
+
+      assert_requested(:post, "#{TEST_SOLR_URL}/update/json", body: JSON.generate({"delete" => id}))
+    end
+
+    it "raises on non-200 http response" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json").to_return(status: 500)
+
+      assert_raises(RuntimeError) do
+        @writer.delete("12345")
+      end
+    end
+  end
+
+  describe "#delete_all!" do
+    it "deletes all" do
+      stub_request(:post, "#{TEST_SOLR_URL}/update/json")
+
+      @writer.delete_all!
+
+      assert_requested(:post, "#{TEST_SOLR_URL}/update/json", body: JSON.generate({"delete" => { "query" => "*:*"}}))
+    end
+  end
+end

data/test/test_helper.rb

@@ -3,6 +3,7 @@ require 'minitest/autorun'
 require 'minitest/spec'
 
 require 'webmock/minitest'
+require "httpx/adapters/webmock"
 
 require 'traject'
 require 'marc'

data/traject.gemspec

@@ -30,15 +30,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
+  spec.add_dependency "httpx", "~> 1.4"
   spec.add_dependency "mutex_m"  # httpclient has an undelcared dep needed in ruby 3.4+, not yet released. https://github.com/nahi/httpclient/pull/455
+  spec.add_dependency "base64" # no longer inlcuded in stdlib
 
-  spec.add_dependency "http", ">= 3.0", "< 6" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
+  spec.add_dependency "http", ">= 3.0", "< 7" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 
-  spec.add_development_dependency 'bundler', '>= 1.7', '< 3'
-
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "rspec-mocks", '~> 3.4'
+  # rspec-mocks no longer includes rspec-core as a dependency, we need to include it.
+  spec.add_development_dependency "rspec-core", '~> 3.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.8.3
+  version: 3.9.1
 platform: ruby
 authors:
 - Jonathan Rochkind
 - Bill Dueber
 bindir: bin
 cert_chain: []
-date: 2025-01-28 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -128,6 +128,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: mutex_m
   requirement: !ruby/object:Gem::Requirement
@@ -142,6 +156,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -151,7 +179,7 @@ dependencies:
         version: '3.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -161,7 +189,7 @@ dependencies:
         version: '3.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: marc-fastxmlwriter
   requirement: !ruby/object:Gem::Requirement
@@ -191,27 +219,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.9'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -225,21 +247,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
-  name: rspec-mocks
+  name: rspec-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -316,6 +338,7 @@ files:
 - lib/traject/null_writer.rb
 - lib/traject/oai_pmh_nokogiri_reader.rb
 - lib/traject/qualified_const_get.rb
+- lib/traject/solr_json_httpx_writer.rb
 - lib/traject/solr_json_writer.rb
 - lib/traject/thread_pool.rb
 - lib/traject/translation_map.rb
@@ -357,6 +380,7 @@ files:
 - test/marc_reader_test.rb
 - test/nokogiri_reader_test.rb
 - test/oai_pmh_nokogiri_reader_test.rb
+- test/solr_json_httpx_writer_test.rb
 - test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc
@@ -428,7 +452,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.7.1
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
   system, focused on library-archives-museums input, and indexing to Solr as output.
@@ -462,6 +486,7 @@ test_files:
 - test/marc_reader_test.rb
 - test/nokogiri_reader_test.rb
 - test/oai_pmh_nokogiri_reader_test.rb
+- test/solr_json_httpx_writer_test.rb
 - test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc