traject 3.7.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ffc677e0ebb13e01b852a1d59ddfdd3cd9906142520e0c296f69ebb0eeb7429
-  data.tar.gz: 61b0e966f6ecd4d27e757e4cfc1057c72ac6deca5ad119c78ff883c246744814
+  metadata.gz: 9e9868f3b83385402413a2fc6c8865dc6ab3dd3776c25c0f1f1b88bf20024005
+  data.tar.gz: f86c298c93905948ca9425983e65811b2091ba39d6105895744df183a7c43695
 SHA512:
-  metadata.gz: 8240b450b27df011c2ff998c24c612f44bdd21a2fde5fbab996ffe509f3fc45cec7a8a8947e385a35d06f5bd8ed19732287e8f2d3dab682cc6295f7320f8dfab
-  data.tar.gz: f5dbcb44edb8d37a4e74cd1255aa1b05b638913337577c92d4fa276150c023b9e29823cc2b20ef050b33879ec63d76194d0202697d7c858a7aacd3d08241dcce
+  metadata.gz: 7d1e1122020632ac10d4da030915e0f710c8dd9bb6e9780089129c1ec7febb76f7b0e23ac4828d1e4860429164fe6052c8db768d478ae6102f4819b8cd512f4d
+  data.tar.gz: '08eead90c2ddfebe141aa4bea2f280878c27af35509a9f5b220d2899fad820f9954dbc30392aa3cfd00b79e77b8f5266c69ac7101fe381d5e777e47b2e12aca7'

data/.github/workflows/ruby.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', 'jruby-9.1', 'jruby-9.2' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', '3.1', 'jruby-9.1', 'jruby-9.2' ]
     name: Ruby ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2

data/CHANGES.md

@@ -6,6 +6,16 @@
 
 *
 
+## 3.8.0
+
+SolrJsonWriter: HTTPClient should use OS certs instead of packaged ones
+
+HTTPClient, for whatever reason, prefers its own packaged certs, which are now years out of date
+and don't work with Let's Encrypt.
+
+This changes the code to prefer the OS certs, which can be overridden by setting
+`solr_json_writer.use_packaged_certs` to `true` or `"true"`. 
+
 ## 3.7.0
 
 * Add two new transformation macros, `Traject::Macros::Transformation.delete_if` and `Traject::Macros::Transformations.select`.

data/Gemfile

@@ -11,3 +11,11 @@ group :debug do
   gem "ruby-debug", :platform => "jruby"
   gem "byebug", :platform => "mri"
 end
+
+# ruby-marc stopped supporting ruby 2.3 and 2.4 in newer 1.x versions,
+# while we would still like to support those old versions. When running
+# CI, run with older ruby-marc that still supports them.
+ruby_version_parts = RUBY_VERSION.split(".")
+if ruby_version_parts[0] == "2" && ruby_version_parts[1].to_i < 5
+  gem "marc", "< 1.2.0"
+end

data/lib/traject/solr_json_writer.rb

@@ -86,6 +86,9 @@ require 'concurrent' # for atomic_fixnum
 # * solr_json_writer.http_client Mainly intended for testing, set your own HTTPClient
 #   or mock object to be used for HTTP.
 #
+# * solr_json_writer.use_packaged_certs: unlikely to be needed, set to true for legacy
+#   behavior, to use packaged HTTPClient gem ssl certs. https://github.com/nahi/httpclient/issues/445
+#
 class Traject::SolrJsonWriter
   include Traject::QualifiedConstGet
 
@@ -118,6 +121,15 @@ class Traject::SolrJsonWriter
       @settings["solr_json_writer.http_client"]
     else
       client = HTTPClient.new
+
+      # By default we'll use teh host OS SSL certs, but you can use
+      # setting solr_json_writer.use_packaged_certs to true or "true"
+      # to go back to previous behavior if you have a perverse reason to.
+      # https://github.com/nahi/httpclient/issues/445
+      unless @settings["solr_json_writer.use_packaged_certs"].to_s == "true"
+        client.ssl_config.set_default_paths
+      end
+
       if @settings["solr_writer.http_timeout"]
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
@@ -431,9 +443,27 @@ class Traject::SolrJsonWriter
     attr_reader :response
 
     def initialize(msg, response = nil) # :nodoc:
+      solr_error = find_solr_error(response)
+      msg += ": #{solr_error}" if solr_error
+
       super(msg)
+
       @response = response
     end
+
+    private
+
+    # If we can get the error out of a JSON response, please do,
+    # to include in error message.
+    def find_solr_error(response)
+      return nil unless response && response.body && response.content_type&.start_with?("application/json")
+
+      parsed = JSON.parse(response.body)
+
+      parsed && parsed.dig("error", "msg")
+    rescue JSON::ParserError
+      return nil
+    end
   end
 
   private

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.7.0"
+  VERSION = "3.8.0"
 end

data/test/solr_json_writer_test.rb

@@ -19,7 +19,7 @@ describe "Traject::SolrJsonWriter" do
   class FakeHTTPClient
     # Always reply with this status, normally 200, can
     # be reset for testing error conditions.
-    attr_accessor :response_status
+    attr_accessor :response_status, :body, :content_type
 
     def initialize(*args)
       @post_args = []
@@ -33,10 +33,7 @@ describe "Traject::SolrJsonWriter" do
         @post_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def get(*args)
@@ -44,10 +41,7 @@ describe "Traject::SolrJsonWriter" do
         @get_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def post_args
@@ -65,6 +59,16 @@ describe "Traject::SolrJsonWriter" do
     # Everything else, just return nil please
     def method_missing(*args)
     end
+
+    private
+
+    def faked_response
+      resp = HTTP::Message.new_response(self.body || "")
+      resp.status = self.response_status
+      resp.content_type = self.content_type if self.content_type
+
+      resp
+    end
   end
 
 
@@ -157,6 +161,26 @@ describe "Traject::SolrJsonWriter" do
     assert_length 1, JSON.parse(individual_update2[1])
   end
 
+  it "includes Solr reported error in base error message" do
+    @writer = create_writer("solr_writer.batch_size" => 1, "solr_writer.max_skipped" => 0)
+    @fake_http_client.response_status = 400
+    @fake_http_client.content_type = "application/json;charset=utf-8"
+    @fake_http_client.body =
+      { "responseHeader"=>{"status"=>400, "QTime"=>0},
+        "error"=>{
+          "metadata"=>["error-class", "org.apache.solr.common.SolrException", "root-error-class", "org.apache.solr.common.SolrException"],
+          "msg"=>"ERROR: this is a solr error",
+          "code"=>400
+        }
+      }.to_json
+
+    error = assert_raises(Traject::SolrJsonWriter::MaxSkippedRecordsExceeded) {
+      @writer.put context_with({"id" => "doc_1", "key" => "value"})
+      @writer.close
+    }
+    assert_match(/ERROR: this is a solr error/, error.message)
+  end
+
   it "can #flush" do
     2.times do |i|
       doc = {"id" => "doc_#{i}", "key" => "value"}

data/traject.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 
-  spec.add_development_dependency 'bundler', '>= 1.7', '< 3'
+  spec.add_development_dependency 'bundler', '~>2.0'
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -167,22 +167,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -402,7 +396,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation