traject 3.6.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e47e6648ed9fc963d18e10c9be48a30273147c4920cb4b7e448d078fd2398ac
-  data.tar.gz: efa549ebcbd87e599b56b955b4bd26422dfe7de67697aed6b39cb421c3b80677
+  metadata.gz: 9e9868f3b83385402413a2fc6c8865dc6ab3dd3776c25c0f1f1b88bf20024005
+  data.tar.gz: f86c298c93905948ca9425983e65811b2091ba39d6105895744df183a7c43695
 SHA512:
-  metadata.gz: 6acdd2b8cfc888b221a1f19cd5197127006be81d0525169d531fc9bf43fe02cc9ec87401e6b2442c57ff0cd483d9884504ac75be92e3718cbbc49208dc97024f
-  data.tar.gz: 30abefa7af9e1c170ae8570aa59b6c571a9acc1eb7b0abf6efd64d97550b678c21c72d76a8156ef3844ab01154111fd3747f96d6346ee8a8d76e747b2cf92e1f
+  metadata.gz: 7d1e1122020632ac10d4da030915e0f710c8dd9bb6e9780089129c1ec7febb76f7b0e23ac4828d1e4860429164fe6052c8db768d478ae6102f4819b8cd512f4d
+  data.tar.gz: '08eead90c2ddfebe141aa4bea2f280878c27af35509a9f5b220d2899fad820f9954dbc30392aa3cfd00b79e77b8f5266c69ac7101fe381d5e777e47b2e12aca7'

data/.github/workflows/ruby.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', 'jruby-9.1', 'jruby-9.2' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', '3.1', 'jruby-9.1', 'jruby-9.2' ]
     name: Ruby ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2

data/CHANGES.md

@@ -1,11 +1,25 @@
 # Changes
 
-## Next
+## NEXT
 
 *
 
 *
 
+## 3.8.0
+
+SolrJsonWriter: HTTPClient should use OS certs instead of packaged ones
+
+HTTPClient, for whatever reason, prefers its own packaged certs, which are now years out of date
+and don't work with Let's Encrypt.
+
+This changes the code to prefer the OS certs, which can be overridden by setting
+`solr_json_writer.use_packaged_certs` to `true` or `"true"`. 
+
+## 3.7.0
+
+* Add two new transformation macros, `Traject::Macros::Transformation.delete_if` and `Traject::Macros::Transformations.select`.
+
 ## 3.6.0
 
 * Tiny backward compat changes for ruby 3.0 compat. https://github.com/traject/traject/pull/263

data/Gemfile

@@ -11,3 +11,11 @@ group :debug do
   gem "ruby-debug", :platform => "jruby"
   gem "byebug", :platform => "mri"
 end
+
+# ruby-marc stopped supporting ruby 2.3 and 2.4 in newer 1.x versions,
+# while we would still like to support those old versions. When running
+# CI, run with older ruby-marc that still supports them.
+ruby_version_parts = RUBY_VERSION.split(".")
+if ruby_version_parts[0] == "2" && ruby_version_parts[1].to_i < 5
+  gem "marc", "< 1.2.0"
+end

data/README.md

@@ -177,6 +177,11 @@ TranslationMap use above is just one example of a transformation macro, that tra
 * `split(" ")`: take values and split them, possibly result in multiple values.
 * `transform(proc)`: transform each existing macro using a proc, kind of like `map`.
    eg `to_field "something", extract_xml("//author"), transform( ->(author) { "#{author.last}, #{author.first}" })
+* `delete_if(["a", "b"])`: remove a value from accumulated values if it is included in the passed in argumet.
+    * Can also take a string, proc or regex as an argument. See [tests](test/indexer/macros/transformation_test.rb) for full functionality.
+* `select(proc)`: selects (keeps) values from accumulated values if proc evaluates to true for specifc value.
+    * Can also take a arrays, sets and regex as an argument. See [tests](test/indexer/macros/transformation_test.rb) for full functionality.
+
 
 You can add on as many transformation macros as you want, they will be applied to output in order.
 

data/lib/traject/macros/marc21_semantics.rb

@@ -327,10 +327,14 @@ module Traject::Macros
       if field008 && field008.length >= 11
         date_type = field008.slice(6)
         date1_str = field008.slice(7,4)
-        date2_str = field008.slice(11, 4) if field008.length > 15
+        if field008.length > 15
+          date2_str = field008.slice(11, 4)
+        else
+          date2_str = date1_str
+        end
 
-        # for date_type q=questionable, we have a range.
-        if (date_type == 'q')
+        # for date_type q=questionable, we expect to have a range.
+        if date_type == 'q' and date1_str != date2_str
           # make unknown digits at the beginning or end of range,
           date1 = date1_str.sub("u", "0").to_i
           date2 = date2_str.sub("u", "9").to_i

data/lib/traject/macros/transformation.rb

@@ -157,6 +157,36 @@ module Traject
           acc.collect! { |v| v.gsub(pattern, replace) }
         end
       end
+
+      # Run ruby `delete_if` on the accumulator for values that include or are equal to arg.
+      # It will also accept an array, set, regex pattern, proc or lambda as an arugment.
+      #
+      # @example
+      #   to_field "creator_facet", extract_marc("100abcdq"), delete_if(/foo/)
+      def delete_if(arg)
+        p = if arg.respond_to? :include?
+              proc { |v| arg.include?(v) }
+            else
+              proc { |v| arg === v }
+            end
+
+        ->(_, acc) { acc.delete_if(&p) }
+      end
+
+      # Run ruby `select!` on the accumulator for values that include or are equal to arg.
+      # It accepts an array, set, regex pattern, proc or lambda as an arugument.
+      #
+      # @example
+      #   to_field "creator_facet", extract_marc("100abcdq"), select(->(v) { v != "foo" })
+      def select(arg)
+        p = if arg.respond_to? :include?
+              proc { |v| arg.include?(v) }
+            else
+              proc { |v| arg === v }
+            end
+
+        ->(_, acc) { acc.select!(&p) }
+      end
     end
   end
 end

data/lib/traject/solr_json_writer.rb

@@ -86,6 +86,9 @@ require 'concurrent' # for atomic_fixnum
 # * solr_json_writer.http_client Mainly intended for testing, set your own HTTPClient
 #   or mock object to be used for HTTP.
 #
+# * solr_json_writer.use_packaged_certs: unlikely to be needed, set to true for legacy
+#   behavior, to use packaged HTTPClient gem ssl certs. https://github.com/nahi/httpclient/issues/445
+#
 class Traject::SolrJsonWriter
   include Traject::QualifiedConstGet
 
@@ -118,6 +121,15 @@ class Traject::SolrJsonWriter
       @settings["solr_json_writer.http_client"]
     else
       client = HTTPClient.new
+
+      # By default we'll use teh host OS SSL certs, but you can use
+      # setting solr_json_writer.use_packaged_certs to true or "true"
+      # to go back to previous behavior if you have a perverse reason to.
+      # https://github.com/nahi/httpclient/issues/445
+      unless @settings["solr_json_writer.use_packaged_certs"].to_s == "true"
+        client.ssl_config.set_default_paths
+      end
+
       if @settings["solr_writer.http_timeout"]
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
@@ -431,9 +443,27 @@ class Traject::SolrJsonWriter
     attr_reader :response
 
     def initialize(msg, response = nil) # :nodoc:
+      solr_error = find_solr_error(response)
+      msg += ": #{solr_error}" if solr_error
+
       super(msg)
+
       @response = response
     end
+
+    private
+
+    # If we can get the error out of a JSON response, please do,
+    # to include in error message.
+    def find_solr_error(response)
+      return nil unless response && response.body && response.content_type&.start_with?("application/json")
+
+      parsed = JSON.parse(response.body)
+
+      parsed && parsed.dig("error", "msg")
+    rescue JSON::ParserError
+      return nil
+    end
   end
 
   private

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.6.0"
+  VERSION = "3.8.0"
 end

data/test/indexer/macros/macros_marc21_semantics_test.rb

@@ -209,6 +209,10 @@ describe "Traject::Macros::Marc21Semantics" do
       @record = MARC::Reader.new(support_file_path  "date_type_r_missing_date2.marc").to_a.first
       assert_equal 1957, Marc21Semantics.publication_date(@record)
     end
+    it "provides a fallback for a missing second date" do
+      @record = MARC::Reader.new(support_file_path  "missing-second-date.marc").to_a.first
+      assert_equal 1678, Marc21Semantics.publication_date(@record)
+    end
 
     it "works correctly with date type 'q'" do
       val = @record['008'].value

data/test/indexer/macros/transformation_test.rb

@@ -174,4 +174,114 @@ describe "Traject::Macros::Transformation" do
     end
   end
 
+  describe "delete_if" do
+
+    describe "argument is an Array" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "filters out selected values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "filters out selected values from accumulatd values" do
+        arg = ->(v) { v == "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
+  describe "select" do
+
+    describe "argument is an Array" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "selects a subset of values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "selects a subset of values from accumulatd values" do
+        arg = ->(v) { v != "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
 end

data/test/solr_json_writer_test.rb

@@ -19,7 +19,7 @@ describe "Traject::SolrJsonWriter" do
   class FakeHTTPClient
     # Always reply with this status, normally 200, can
     # be reset for testing error conditions.
-    attr_accessor :response_status
+    attr_accessor :response_status, :body, :content_type
 
     def initialize(*args)
       @post_args = []
@@ -33,10 +33,7 @@ describe "Traject::SolrJsonWriter" do
         @post_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def get(*args)
@@ -44,10 +41,7 @@ describe "Traject::SolrJsonWriter" do
         @get_args << args
       end
 
-      resp = HTTP::Message.new_response("")
-      resp.status = self.response_status
-
-      return resp
+      return faked_response
     end
 
     def post_args
@@ -65,6 +59,16 @@ describe "Traject::SolrJsonWriter" do
     # Everything else, just return nil please
     def method_missing(*args)
     end
+
+    private
+
+    def faked_response
+      resp = HTTP::Message.new_response(self.body || "")
+      resp.status = self.response_status
+      resp.content_type = self.content_type if self.content_type
+
+      resp
+    end
   end
 
 
@@ -157,6 +161,26 @@ describe "Traject::SolrJsonWriter" do
     assert_length 1, JSON.parse(individual_update2[1])
   end
 
+  it "includes Solr reported error in base error message" do
+    @writer = create_writer("solr_writer.batch_size" => 1, "solr_writer.max_skipped" => 0)
+    @fake_http_client.response_status = 400
+    @fake_http_client.content_type = "application/json;charset=utf-8"
+    @fake_http_client.body =
+      { "responseHeader"=>{"status"=>400, "QTime"=>0},
+        "error"=>{
+          "metadata"=>["error-class", "org.apache.solr.common.SolrException", "root-error-class", "org.apache.solr.common.SolrException"],
+          "msg"=>"ERROR: this is a solr error",
+          "code"=>400
+        }
+      }.to_json
+
+    error = assert_raises(Traject::SolrJsonWriter::MaxSkippedRecordsExceeded) {
+      @writer.put context_with({"id" => "doc_1", "key" => "value"})
+      @writer.close
+    }
+    assert_match(/ERROR: this is a solr error/, error.message)
+  end
+
   it "can #flush" do
     2.times do |i|
       doc = {"id" => "doc_#{i}", "key" => "value"}

data/test/test_support/missing-second-date.marc

@@ -0,0 +1 @@
+01351nem a2200313 a 4500001001100000001001100011008004100022034001300063035002200076043003000098080001200128100006100140245011600201255002900317260005900346300003500405500004500440500013600485500003800621500005000659530004400709651006000753700003800813710010200851730000900953856005300962907001401015940000801029.b20028118.b6928510x170714q1678    fr ||||   |  |||| ||fre|c1 aab10000  a(OCoLC)1120596466  ae-sp---be-spcce2catmarc  a(084.3)1 aBeaulieu, Sébastien de Pontault,csieur de,d1613-1674.10aPlan de la ville de Puiçerdah[Document cartogràfic] :bpris en 1678 /c[Beaulieu] ; DR f. [Des Roches fecit]  aEscala [1:10 000 aprox.]  a[A Paris :bpar le Chevalier de Beaulieu,cpost. 1678]  a1 mapa :bgravat;c28 x 32 cm.  aEscala gràfica: Eschelle de 150 toises.  aPertany a l'obra "Les plans et profils des principales villes et lieux considerables de la Principauté de Catalogne", de Beaulieu.  aPeu d'impremta de l'obra general.  aMapa emmarcat en una orla amb motius florals.  aTambé disponible la versió en línia. 4aPuigcerdà (Catalunya)xMapesxObres anteriors al 1800.1 aDes Roches, Jean Baptiste Hamont.2 aCol·lecció de mapes antics de Martí Gelabertó (Universitat Autònoma de Barcelona)5ES-BaUAB.0 aDDD.41zAccés lliureuhttps://ddd.uab.cat/record/180313  ab20028118  aUAB

data/traject.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "concurrent-ruby", ">= 0.8.0"
   spec.add_dependency "marc", "~> 1.0"
 
-  spec.add_dependency "hashie", ">= 3.1", "< 5" # used for Indexer#settings
+  spec.add_dependency "hashie", ">= 3.1", "< 6" # used for Indexer#settings
   spec.add_dependency "slop", "~> 4.0" # command line parsing
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 
-  spec.add_development_dependency 'bundler', '>= 1.7', '< 3'
+  spec.add_development_dependency 'bundler', '~>2.0'
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2022-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -48,7 +48,7 @@ dependencies:
         version: '3.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
         version: '3.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -167,22 +167,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.7'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -350,6 +344,7 @@ files:
 - test/test_support/manufacturing_consent.marc
 - test/test_support/manuscript_online_thesis.marc
 - test/test_support/microform_online_conference.marc
+- test/test_support/missing-second-date.marc
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc
@@ -401,7 +396,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
@@ -455,6 +450,7 @@ test_files:
 - test/test_support/manufacturing_consent.marc
 - test/test_support/manuscript_online_thesis.marc
 - test/test_support/microform_online_conference.marc
+- test/test_support/missing-second-date.marc
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc