traject 3.7.0 → 3.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ffc677e0ebb13e01b852a1d59ddfdd3cd9906142520e0c296f69ebb0eeb7429
-  data.tar.gz: 61b0e966f6ecd4d27e757e4cfc1057c72ac6deca5ad119c78ff883c246744814
+  metadata.gz: 0e54c076cdd70f6beacb8e9bdb6057c6ba809393398c7c786c6e7c943117581e
+  data.tar.gz: 77ed56fc2ee2355ab584af074fee656f8cfb543d4275a63384ce069a52a20f1a
 SHA512:
-  metadata.gz: 8240b450b27df011c2ff998c24c612f44bdd21a2fde5fbab996ffe509f3fc45cec7a8a8947e385a35d06f5bd8ed19732287e8f2d3dab682cc6295f7320f8dfab
-  data.tar.gz: f5dbcb44edb8d37a4e74cd1255aa1b05b638913337577c92d4fa276150c023b9e29823cc2b20ef050b33879ec63d76194d0202697d7c858a7aacd3d08241dcce
+  metadata.gz: 3b3a6826b6825c08604c7ed8608b63fe98a889de676c2b0b7ba7eb1002eb029fe04f19b4fa57a480ff486f0acd911ddec749c2f6a3589581588d17e93648deeb
+  data.tar.gz: 1451ae18ada4181224d4a02269ba2532b0914d69ab7d07ab60d3390e12605c239e0304a93511ab3c3c854d9251623af94c852b2a97eab02fb19f969f43a374cb

data/.github/workflows/ruby.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', 'jruby-9.1', 'jruby-9.2' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', '3.1', 'jruby-9.1', 'jruby-9.2' ]
     name: Ruby ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2

data/CHANGES.md

@@ -6,6 +6,20 @@
 
 *
 
+## 3.8.1
+
+Ugh. Forgot about Jruby 9.1 problem with bundler 2. Changing the requirement back. 
+
+## 3.8.0
+
+SolrJsonWriter: HTTPClient should use OS certs instead of packaged ones
+
+HTTPClient, for whatever reason, prefers its own packaged certs, which are now years out of date
+and don't work with Let's Encrypt.
+
+This changes the code to prefer the OS certs, which can be overridden by setting
+`solr_json_writer.use_packaged_certs` to `true` or `"true"`. 
+
 ## 3.7.0
 
 * Add two new transformation macros, `Traject::Macros::Transformation.delete_if` and `Traject::Macros::Transformations.select`.

data/Gemfile

@@ -11,3 +11,11 @@ group :debug do
   gem "ruby-debug", :platform => "jruby"
   gem "byebug", :platform => "mri"
 end
+
+# ruby-marc stopped supporting ruby 2.3 and 2.4 in newer 1.x versions,
+# while we would still like to support those old versions. When running
+# CI, run with older ruby-marc that still supports them.
+ruby_version_parts = RUBY_VERSION.split(".")
+if ruby_version_parts[0] == "2" && ruby_version_parts[1].to_i < 5
+  gem "marc", "< 1.2.0"
+end

data/lib/traject/solr_json_writer.rb

@@ -86,6 +86,9 @@ require 'concurrent' # for atomic_fixnum
 # * solr_json_writer.http_client Mainly intended for testing, set your own HTTPClient
 #   or mock object to be used for HTTP.
 #
+# * solr_json_writer.use_packaged_certs: unlikely to be needed, set to true for legacy
+#   behavior, to use packaged HTTPClient gem ssl certs. https://github.com/nahi/httpclient/issues/445
+#
 class Traject::SolrJsonWriter
   include Traject::QualifiedConstGet
 
@@ -118,6 +121,15 @@ class Traject::SolrJsonWriter
       @settings["solr_json_writer.http_client"]
     else
       client = HTTPClient.new
+
+      # By default we'll use teh host OS SSL certs, but you can use
+      # setting solr_json_writer.use_packaged_certs to true or "true"
+      # to go back to previous behavior if you have a perverse reason to.
+      # https://github.com/nahi/httpclient/issues/445
+      unless @settings["solr_json_writer.use_packaged_certs"].to_s == "true"
+        client.ssl_config.set_default_paths
+      end
+
       if @settings["solr_writer.http_timeout"]
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
@@ -431,9 +443,27 @@ class Traject::SolrJsonWriter
     attr_reader :response
 
     def initialize(msg, response = nil) # :nodoc:
+      solr_error = find_solr_error(response)
+      msg += ": #{solr_error}" if solr_error
+
       super(msg)
+
       @response = response
     end
+
+    private
+
+    # If we can get the error out of a JSON response, please do,
+    # to include in error message.
+    def find_solr_error(response)
+      return nil unless response && response.body && response.content_type&.start_with?("application/json")
+
+      parsed = JSON.parse(response.body)
+
+      parsed && parsed.dig("error", "msg")
+    rescue JSON::ParserError
+      return nil
+    end
   end
 
   private

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.7.0"
+  VERSION = "3.8.1"
 end

data/test/solr_json_writer_test.rb

@@ -19,7 +19,7 @@ describe "Traject::SolrJsonWriter" do
   class FakeHTTPClient
     # Always reply with this status, normally 200, can
     # be reset for testing error conditions.
-    attr_accessor :response_status
+    attr_accessor :response_status, :body, :content_type
 
     def initialize(*args)
       @post_args = []
@@ -33,10 +33,7 @@ describe "Traject::SolrJsonWriter" do
         @post_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def get(*args)
@@ -44,10 +41,7 @@ describe "Traject::SolrJsonWriter" do
         @get_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def post_args
@@ -65,6 +59,16 @@ describe "Traject::SolrJsonWriter" do
     # Everything else, just return nil please
     def method_missing(*args)
     end
+
+    private
+
+    def faked_response
+      resp = HTTP::Message.new_response(self.body || "")
+      resp.status = self.response_status
+      resp.content_type = self.content_type if self.content_type
+
+      resp
+    end
   end
 
 
@@ -157,6 +161,26 @@ describe "Traject::SolrJsonWriter" do
     assert_length 1, JSON.parse(individual_update2[1])
   end
 
+  it "includes Solr reported error in base error message" do
+    @writer = create_writer("solr_writer.batch_size" => 1, "solr_writer.max_skipped" => 0)
+    @fake_http_client.response_status = 400
+    @fake_http_client.content_type = "application/json;charset=utf-8"
+    @fake_http_client.body =
+      { "responseHeader"=>{"status"=>400, "QTime"=>0},
+        "error"=>{
+          "metadata"=>["error-class", "org.apache.solr.common.SolrException", "root-error-class", "org.apache.solr.common.SolrException"],
+          "msg"=>"ERROR: this is a solr error",
+          "code"=>400
+        }
+      }.to_json
+
+    error = assert_raises(Traject::SolrJsonWriter::MaxSkippedRecordsExceeded) {
+      @writer.put context_with({"id" => "doc_1", "key" => "value"})
+      @writer.close
+    }
+    assert_match(/ERROR: this is a solr error/, error.message)
+  end
+
   it "can #flush" do
     2.times do |i|
       doc = {"id" => "doc_#{i}", "key" => "value"}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.8.1
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -402,7 +402,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation