traject 3.8.3 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4434e6a85dc5176d84dfb2c6e42c0882937cdffa37c24e5f5f06844ad0e542d
-  data.tar.gz: fd2da1d197ace61ab86e2308298e77ed64866ef0f1ffc49b88b344578900f76e
+  metadata.gz: 9e73d6d38cf8601c13b83efcbafd739f6f387b00d8982ee1eca95230d2361b37
+  data.tar.gz: c9a516522aa1f17e612ea4fbff0f9e3e99d80ae841bc0b43e3320038311cac3e
 SHA512:
-  metadata.gz: 3aad25487edbb9489a30f6d8a9c6c2fdb467683c6cd6a96199d7de10538f861fd92c2f25ec8213b5a753ef85bc7440c1d27ee48d65fc05d2f36e735a15a7b6a6
-  data.tar.gz: 7641b36d876e01d3b8188add71a4121a768116a8bd2462c1fcfb0e2652af7ddebfc9a17d03dd24c68fa56b8810e2b7cd865eea53ff905a53fb5f7675c5dc99e4
+  metadata.gz: 23a439be2faf8245af5eaf2d653cb61b7bc5a66e43a251df793c34b69276ef2dc3c672bb3e9b23ea3a02e2c2512c1fd759844018bac15da19656a19614dd88fd
+  data.tar.gz: 257fe8d4015d16d96d1beabde4c784e849f97d519dee68a59391c562b6e1f7a4d058b4c14f31b05b61bcf2f39349bf6e36057ef86f6d96ce5e5827a6b540dec9

data/.github/workflows/ruby.yml

@@ -19,8 +19,9 @@ jobs:
           - '3.2'
           - '3.3'
           - '3.4'
-          - 'jruby-9.3'
-          - 'jruby-9.4'
+          - '4.0'
+          - 'jruby-10.0'
+          - 'jruby-10.1'
     name: Ruby ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2

data/CHANGES.md

@@ -1,26 +1,47 @@
 # Changes
 
-## 3.8.3 Compatibility release for ruby 3.4
+## Next
+
+Work will continue to lessen dependence on `http-client` (which is
+unsupported) while making sure to not break anything or lose
+important characteristics like keepAlive and multi threading.
+
+##
+
+## 3.9.0
+
+* add SolrJsonWriter2 using httpx client instead of httpclient [@jrochkind] https://github.com/traject/traject/pull/307
+* Eliminate frozen string literal warnings in latest rubies. [@jrochkind]
+* Update marc language and geographic maps from LOC [@maxkadel]
+* Support http gem 6.x [@cbeer]
+* Replace CGI.* methods removed from ruby 4.0 with URI.* methods [@jrochkind]
+
+## 3.8.3
+
+Compatibility release for ruby 3.4
+
+[Incorporates work from @jrochkind and @jcoyne]
 
 Note that some gems, `ruby-marc` in particular, will throw zillions of
-warnings about literal strings being frozen in future versions of ruby. 
+warnings about literal strings being frozen in future versions of ruby
+when running under ruby 3.4.
 
 - Update CI testing matrix
   - removed MRI ruby 2.4, 2.5, 2.6, and jruby-9.2
   - added MRI ruby 3.4
-- Add gem dependencies for `csv` and `mutex_m`, both of which were 
+- Add gem dependencies for `csv` and `mutex_m`, both of which were
   part of the standard library prior to ruby 3.4
 
 
 ## 3.8.2
 
-Bug fix for the `#filing_version` logic, which was incorrectly assuming the 
+Bug fix for the `#filing_version` logic, which was incorrectly assuming the
 first subfield in a field would hold content (e.g., `$a`) and thus failed
 when it held a pointer to a linking field (e.g., `$6 245-01`)
 
 ## 3.8.1
 
-Ugh. Forgot about Jruby 9.1 problem with bundler 2. Changing the requirement back. 
+Ugh. Forgot about Jruby 9.1 problem with bundler 2. Changing the requirement back.
 
 ## 3.8.0
 
@@ -30,7 +51,7 @@ HTTPClient, for whatever reason, prefers its own packaged certs, which are now y
 and don't work with Let's Encrypt.
 
 This changes the code to prefer the OS certs, which can be overridden by setting
-`solr_json_writer.use_packaged_certs` to `true` or `"true"`. 
+`solr_json_writer.use_packaged_certs` to `true` or `"true"`.
 
 ## 3.7.0
 

data/README.md

@@ -49,6 +49,7 @@ You can keep your settings and indexing rules in one config file, or split them
 
 There are two main categories of directives in your configuration files: _Settings_, and _Indexing Rules_.
 
+
 ## Settings
 
 Settings are a flat list of key/value pairs, where the keys are always strings and the values usually are too. They look like this in a config file:
@@ -86,6 +87,11 @@ You can also use `store` if you want to force-set: last set wins.
 See, docs page on [Settings](./doc/settings.md) for list
 of all standardized settings.
 
+## Note: New writer coming
+
+The default writer, `Traject::SolrJsonWriter` uses the [http gem](https://github.com/httprb/http) for HTTP.  We will be in the future switching to use [httpx](https://github.com/HoneyryderChuck/httpx) for the default writer.
+
+To opt into the new httpx-based writer now, either set the shell ENV variable `TRAJECT_DEFAULT_WRITER=SolrJsonHttpxWriter`, or set configuration `provide "writer_class_name", "Traject::SolrJsonHttpxWriter"`
 
 ## Indexing rules: 'to_field'
 

data/lib/tasks/load_maps.rake

@@ -12,14 +12,13 @@ namespace :load_maps do
   task :marc_geographic do |task|
     require_nokogiri(task)
 
-    source_url = "http://www.loc.gov/marc/geoareas/gacs_code.html"
+    source_url = "https://www.loc.gov/marc/geoareas/gacs_code.html"
 
     filename = ENV["OUTPUT_TO"] || File.expand_path("../../translation_maps/marc_geographic.yaml", __FILE__)
     file = File.open( filename, "w:utf-8" )
 
     $stderr.puts "Writing to `#{filename}` ..."
-
-    html = Nokogiri::HTML(open(source_url).read)
+    html = Nokogiri::HTML(Net::HTTP.get(URI.parse(source_url)))
 
     file.puts "# Translation map for marc geographic codes constructed by `rake load_maps:marc_geographic` task"    
     file.puts "# Scraped from #{source_url} at #{Time.now}"
@@ -51,6 +50,7 @@ namespace :load_maps do
     file.puts("# Map Language Codes (in 008[35-37], 041) to User Friendly Term\r")
 
     marc_language_source_url = 'https://www.loc.gov/standards/codelists/languages.xml'
+    file.puts "# Scraped from #{marc_language_source_url} at #{Time.now}"
     doc = Nokogiri::XML(URI.parse(marc_language_source_url).open)
     marc_language_hash = doc.xpath('//codelist:language', codelist: CODELIST_NS)
                             .to_h do |node|

data/lib/traject/csv_writer.rb

@@ -5,7 +5,6 @@ require 'csv'
 # Use DelimitedWriter for non-CSV lines (e.g., tab-delimited)
 #
 #
-
 class Traject::CSVWriter < Traject::DelimitedWriter
 
   def initialize(*args)
@@ -26,6 +25,11 @@ class Traject::CSVWriter < Traject::DelimitedWriter
   # Let CSV take care of the comma escaping
   def escape(x)
     x = x.to_s
+
+    if x.frozen?
+      x = x.dup
+    end
+
     x.gsub! internal_delimiter, @eidelim
     x
   end

data/lib/traject/delimited_writer.rb

@@ -20,7 +20,7 @@ require 'traject/line_writer'
 # escape delimiters/internal_delimiters in the following way:
 #  * If the delimiter is a tab, replace tabs in values with a single space
 #  * If the delimiter is anything else, prefix it with a backslash
-
+#
 class Traject::DelimitedWriter < Traject::LineWriter
 
   attr_reader :delimiter,  :internal_delimiter, :edelim, :eidelim
@@ -83,6 +83,11 @@ class Traject::DelimitedWriter < Traject::LineWriter
   # Escape the delimiters in whatever way has been defined
   def escape(x)
     x = x.to_s
+
+    if x.frozen?
+      x = x.dup
+    end
+
     x.gsub! @delimiter, @edelim if @delimiter
     x.gsub! @internal_delimiter, @eidelim
     x

data/lib/traject/experimental_nokogiri_streaming_reader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Traject
   # An EXPERIMENTAL HALF-FINISHED implementation of a streaming/pull reader using Nokogiri.
   # Not ready for use, not stable API, could go away.
@@ -134,7 +136,7 @@ module Traject
         @inverted_namespaces  = namespaces.invert
         @clipboard = clipboard
         # We're guessing using a string will be more efficient than an array
-        @current_path         = ""
+        @current_path         = +""
         @floating             = false
 
         @path_spec, @floating = parse_path(str_spec)

data/lib/traject/indexer/context.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Represents the context of a specific record being indexed, passed
 # to indexing logic blocks
 #
@@ -57,7 +59,7 @@ class Traject::Indexer
     # a string label that can be used to refer to a particular record in log messages and
     # exceptions. Includes various parts depending on what we got.
     def record_inspect
-      str = "<"
+      str = +"<"
 
       str << "record ##{position}" if position
 

data/lib/traject/indexer.rb

@@ -10,6 +10,7 @@ require 'traject/indexer/step'
 require 'traject/marc_reader'
 require 'traject/json_writer'
 require 'traject/solr_json_writer'
+require 'traject/solr_json_httpx_writer'
 require 'traject/debug_writer'
 require 'traject/array_writer'
 
@@ -282,7 +283,7 @@ class Traject::Indexer
   def self.default_settings
     @default_settings ||= {
         # Writer defaults
-        "writer_class_name"       => "Traject::SolrJsonWriter",
+        "writer_class_name"       => ENV['TRAJECT_DEFAULT_WRITER'] == 'SolrJsonHttpxWriter' ? "Traject::SolrJsonHttpxWriter" : "Traject::SolrJsonWriter2",
         "solr_writer.batch_size"  => 100,
         "solr_writer.thread_pool" => 1,
 

data/lib/traject/macros/marc21_semantics.rb

@@ -1,4 +1,5 @@
 # Encoding: UTF-8
+# frozen_string_literal: true
 
 require 'traject/marc_extractor'
 
@@ -570,7 +571,7 @@ module Traject::Macros
     # associated with the content of subfield $v, $x, $y, and $z."
     # http://www.loc.gov/marc/bibliographic/bd600.html
     def self.assemble_lcsh(marc_field, subd_separator = " — ", other_separator = " ")
-      str = ""
+      str = +""
       subd_prefix_codes = %w{v x y z}
 
 

data/lib/traject/oai_pmh_nokogiri_reader.rb

@@ -1,5 +1,4 @@
 require 'uri'
-require 'cgi'
 require 'http'
 
 module Traject
@@ -54,7 +53,7 @@ module Traject
     end
 
     def start_url_verb
-      @start_url_verb ||= (array = CGI.parse(URI.parse(start_url).query)["verb"]) && array.first
+      @start_url_verb ||= URI.decode_www_form(URI.parse(start_url).query).to_h["verb"]
     end
 
     def extra_xpath_hooks
@@ -94,7 +93,7 @@ module Traject
       # resumption URL is just original verb with resumption token, that seems to be
       # the oai-pmh spec.
       parsed_uri = URI.parse(start_url)
-      parsed_uri.query = "verb=#{CGI.escape start_url_verb}&resumptionToken=#{CGI.escape resumption_token}"
+      parsed_uri.query = "verb=#{URI.encode_www_form_component start_url_verb}&resumptionToken=#{URI.encode_www_form_component resumption_token}"
       parsed_uri.to_s
     end