traject 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bc0afc820efb8a2479d96913ae67552556978d264e20a55d0a4d4cef5ff9a2a
-  data.tar.gz: 773529d885d46d2ba0afd323d680724678225bd3616a720dba92809479395dce
+  metadata.gz: 9a69bdb470c759d08d117f910ec1c6b16cbe93dbb3e955b653a07e3c03efec68
+  data.tar.gz: 68016ddf17fe29348248e5f26ff5a813c1485c37a0ac014faa69803be182f7b4
 SHA512:
-  metadata.gz: 395f1a3cf62cfd0cccbdcd428830feb098586c9ca1edc7adecff31483545a62dbc06631e7c166ea0ce1b229afeddcec7d994245d78871f1a6644c591d6cc432c
-  data.tar.gz: 647f9cfbf0a7a0876cef29b5a90a9fd6d3dd1f6a1010b332d58e132875b5c4a7f8be9bdab1afabb5ec0bc583af9ea0d8a422cc87eec6c04d93c9fe9f4b4d1e9a
+  metadata.gz: 8921a5d2349025291f28c7d39ebec44c0c0ebdc11a489e1d4c22913c04c840cf40163c576b14432794f1d663b9b81eb1d777b87d38af2b429f4b6ac98982ad48
+  data.tar.gz: 8bdb95f98baa11ee60c74a7a4bbab06c44fb5da466138d098b05e5ab3fddc28a1fe8926044be5350e5e0b8a91ccd2d6e2e2ecf78424cfc0430464dd0ccf79c66

data/CHANGES.md

@@ -1,5 +1,24 @@
 # Changes
 
+## Next
+
+*
+
+*
+
+*
+
+## 3.2.0
+
+* NokogiriReader has a "nokogiri.strict_mode" setting. Set to true or string 'true' to ask Nokogori to parse in strict mode, so it will immediately raise on ill-formed XML, instead of nokogiri's default to do what it can with it. https://github.com/traject/traject/pull/226
+
+* SolrJsonWriter
+
+  * Utility method `delete_all!` sends a delete all query to the Solr URL endpoint. https://github.com/traject/traject/pull/227
+
+  * Allow basic auth configuration of the default http client via `solr_writer.basic_auth_user` and `solr_writer.basic_auth_password`. https://github.com/traject/traject/pull/231
+
+
 ## 3.1.0
 
 ### Added

data/doc/settings.md

@@ -93,6 +93,8 @@ settings are applied first of all. It's recommended you use `provide`.
 
 * `solr_writer.thread_pool`: defaults to 1 (single bg thread). A thread pool is used for submitting docs to solr. Set to 0 or nil to disable threading. Set to 1, there will still be a single bg thread doing the adds. May make sense to set higher than number of cores on your indexing machine, as these threads will mostly be waiting on Solr. Speed/capacity of your solr might be more relevant. Note that processing_thread_pool threads can end up submitting to solr too, if solr_json_writer.thread_pool is full.
 
+* `solr_writer.basic_auth_user`, `solr_writer.basic_auth_password`: Not set by default but when both are set the default writer is configured with basic auth.
+
 
 ### Dealing with MARC data
 

data/lib/traject/nokogiri_reader.rb

@@ -21,6 +21,9 @@ module Traject
   #   If you need to use namespaces here, you need to have them registered with
   #   `nokogiri.default_namespaces`. If your source docs use namespaces, you DO need
   #   to use them in your each_record_xpath.
+  # * nokogiri.strict_mode: if set to `true` or `"true"`, ask Nokogiri to parse in 'strict'
+  #   mode, it will raise a `Nokogiri::XML::SyntaxError` if the XML is not well-formed, instead
+  #   of trying to take it's best-guess correction. https://nokogiri.org/tutorials/ensuring_well_formed_markup.html
   # * nokogiri_reader.extra_xpath_hooks: Experimental in progress, see below.
   #
   # ## nokogiri_reader.extra_xpath_hooks: For handling nodes outside of your each_record_xpath
@@ -87,7 +90,11 @@ module Traject
     end
 
     def each
-      whole_input_doc = Nokogiri::XML.parse(input_stream)
+      config_proc = if settings["nokogiri.strict_mode"]
+        proc { |config| config.strict }
+      end
+
+      whole_input_doc = Nokogiri::XML.parse(input_stream, &config_proc)
 
       if each_record_xpath
         whole_input_doc.xpath(each_record_xpath, default_namespaces).each do |matching_node|

data/lib/traject/solr_json_writer.rb

@@ -114,6 +114,12 @@ class Traject::SolrJsonWriter
       if @settings["solr_writer.http_timeout"]
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
+
+      if @settings["solr_writer.basic_auth_user"] &&
+          @settings["solr_writer.basic_auth_password"]
+        client.set_auth(@settings["solr.url"], @settings["solr_writer.basic_auth_user"], @settings["solr_writer.basic_auth_password"])
+      end
+
       client
     end
 
@@ -270,6 +276,13 @@ class Traject::SolrJsonWriter
     end
   end
 
+  # Send a delete all query.
+  #
+  # This method takes no params and will not automatically commit the deletes.
+  # @example @writer.delete_all!
+  def delete_all!
+    delete(query: "*:*")
+  end
 
   # Get the logger from the settings, or default to an effectively null logger
   def logger

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.1.0"
+  VERSION = "3.2.0"
 end

data/test/nokogiri_reader_test.rb

@@ -134,6 +134,16 @@ describe "Traject::NokogiriReader" do
     end
   end
 
+  describe "strict_mode" do
+    it "raises on non-well-formed" do
+      # invalid because two sibling root nodes, XML requiers one root node
+      reader = Traject::NokogiriReader.new(StringIO.new("<doc></doc><doc></doc>"), {"nokogiri.strict_mode" => "true" })
+      assert_raises(Nokogiri::XML::SyntaxError) {
+        reader.each { |r| }
+      }
+    end
+  end
+
 
   def shared_tests
     @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {

data/test/solr_json_writer_test.rb

@@ -170,6 +170,27 @@ describe "Traject::SolrJsonWriter" do
     assert_length 1, @fake_http_client.post_args, "Has flushed to solr"
   end
 
+  it "defaults to not setting basic authentication" do
+    settings = { "solr.url" => "http://example.com/solr/foo" }
+    writer = Traject::SolrJsonWriter.new(settings)
+    auth = writer.instance_variable_get("@http_client")
+      .www_auth.basic_auth.instance_variable_get("@auth")
+    assert(auth.empty?)
+  end
+
+  it "allows basic authentication setup" do
+    settings = {
+      "solr.url" => "http://example.com/solr/foo",
+      "solr_writer.basic_auth_user" => "foo",
+      "solr_writer.basic_auth_password" => "bar",
+    }
+
+    writer = Traject::SolrJsonWriter.new(settings)
+    auth = writer.instance_variable_get("@http_client")
+      .www_auth.basic_auth.instance_variable_get("@auth")
+    assert(!auth.empty?)
+  end
+
   describe "commit" do
     it "commits on close when set" do
       @writer = create_writer("solr.url" => "http://example.com", "solr_writer.commit_on_close" => "true")
@@ -365,4 +386,13 @@ describe "Traject::SolrJsonWriter" do
       end
     end
   end
+
+  describe "#delete_all!" do
+    it "deletes all" do
+      @writer.delete_all!
+      post_args = @fake_http_client.post_args.first
+      assert_equal "http://example.com/solr/update/json", post_args[0]
+      assert_equal JSON.generate({"delete" => { "query" => "*:*"}}), post_args[1]
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-22 00:00:00.000000000 Z
+date: 2019-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby