traject 3.5.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01ca968682bb3fc2a8313131ef6344bfc9e5418b767b2900c3d799caa356d016
-  data.tar.gz: a3fd6c9a3bec88c6ba592500ea170357b059533f28c5ba3fb2fe72de39702a2a
+  metadata.gz: 2e47e6648ed9fc963d18e10c9be48a30273147c4920cb4b7e448d078fd2398ac
+  data.tar.gz: efa549ebcbd87e599b56b955b4bd26422dfe7de67697aed6b39cb421c3b80677
 SHA512:
-  metadata.gz: 93547927e90b7947588c1983bd37de4651722bebaff7aaad2d3965ec46eed8c647971f1ce093beb86a5c47c2efa999516d00fb6956682c98913cd54ea5a1a2b8
-  data.tar.gz: 128ea6e2517711f2324541215f814430f963b0042ae321ebc3f29646398990dc09d2a307acc32fb89a70142d74763fdfee6a61d220043622b5b8b11fbcd645d8
+  metadata.gz: 6acdd2b8cfc888b221a1f19cd5197127006be81d0525169d531fc9bf43fe02cc9ec87401e6b2442c57ff0cd483d9884504ac75be92e3718cbbc49208dc97024f
+  data.tar.gz: 30abefa7af9e1c170ae8570aa59b6c571a9acc1eb7b0abf6efd64d97550b678c21c72d76a8156ef3844ab01154111fd3747f96d6346ee8a8d76e747b2cf92e1f

data/.github/workflows/ruby.yml

@@ -12,14 +12,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', 'jruby-9.1', 'jruby-9.2' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', 'jruby-9.1', 'jruby-9.2' ]
     name: Ruby ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+
+    - name: set JAVA_OPTS for jruby-9.1
+      run: echo 'JAVA_OPTS="--add-opens java.base/java.security.cert=ALL-UNNAMED --add-opens java.base/java.security=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"' >> $GITHUB_ENV
+      if: ${{ matrix.ruby == 'jruby-9.1' }}
+      # https://github.com/jruby/jruby/issues/4834
+      # Still seems to be an issue in jruby-9.1, but not 9.2
+      # https://github.community/t/conditional-setting-of-env-variables-in-gh-actions/179650
+
     - name: Install dependencies
       run: bundle install --jobs 4 --retry 3
     - name: Run tests

data/CHANGES.md

@@ -6,7 +6,11 @@
 
 *
 
-*
+## 3.6.0
+
+* Tiny backward compat changes for ruby 3.0 compat. https://github.com/traject/traject/pull/263
+
+* Allow gem `http` 5.x in gemspec. https://github.com/traject/traject/pull/269
 
 ## 3.5.0
 

data/README.md

@@ -8,7 +8,7 @@ Traject can also be generalized to a set of tools for getting structured data fr
 
 **Traject is stable, mature software, that is already being used in production by its authors and several other institutions.**
 
-[![Gem Version](https://badge.fury.io/rb/traject.png)](http://badge.fury.io/rb/traject)
+[![Gem Version](https://badge.fury.io/rb/traject.svg)](http://badge.fury.io/rb/traject)
 [![CI Status](https://github.com/traject/traject/workflows/CI/badge.svg?branch=master)](https://github.com/traject/traject/actions?query=workflow%3ACI+branch%3Amaster)
 
 
@@ -468,6 +468,22 @@ Also see `-I load_path` option and suggestions for Bundler use under Extending W
 See also [Hints for batch and cronjob use](./doc/batch_execution.md) of traject.
 
 
+## A small but complete example
+
+To process a MARC XML file with the data shown in [./examples/marc/tiny.xml](./examples/marc/tiny.xml) you can use save the following configuration as `config.rb`:
+
+```
+to_field 'title', extract_marc('245a', first: true)
+```
+
+and run Traject as follows:
+
+```
+traject -t xml -c config.rb -w Traject::DebugWriter tiny.xml
+```
+
+`-t xml` indicates that the file is a MARC XML file. `-w Traject::DebugWriter` outputs the results to the console (e.g. without saving to Solr).
+
 ## Extending With Your Own Code
 
 Traject config files are full live ruby files, where you can do anything,

data/doc/xml.md

@@ -4,6 +4,8 @@ The [NokogiriIndexer](../lib/traject/nokogiri_indexer.md) is a Traject::Indexer
 
 It by default uses the NokogiriReader to read XML and read Nokogiri::XML::Documents, and includes the NokogiriMacros mix-in, with some macros for operating on Nokogiri::XML::Documents.
 
+Plese notice that the recommened mechanism to parse MARC XML files with Traject is via the `-t` parameter (or the via the `provide "marc_source.type", "xml"` setting). The documentation in this page is for those parsing other (non MARC) XML files.
+
 ## On the command-line
 
 You can tell the traject command-line to use the NokogiriIndexer with the `-i xml` flag:

data/examples/marc/tiny.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<collection xmlns="http://www.loc.gov/MARC21/slim" xmlns:marc="http://www.loc.gov/MARC21/slim">
+ <record>
+  <leader>01352cam a2200349 a 4500</leader>
+  <datafield tag="245" ind1="0" ind2="0">
+   <subfield code="6">880-01</subfield>
+   <subfield code="a">Kazoku kankei no shakai shinrigaku /</subfield>
+   <subfield code="c">Osada Masayoshi hen.</subfield>
+  </datafield>
+ </record>
+ <record>
+  <leader>01121ccm a2200289z  4500</leader>
+  <datafield tag="245" ind1="1" ind2="0">
+   <subfield code="a">Powhatan&#39;s daughter :</subfield>
+   <subfield code="b">march</subfield>
+  </datafield>
+  <datafield tag="100" ind1="1" ind2=" ">
+   <subfield code="a">Sousa, John Philip,</subfield>
+   <subfield code="d">1854-1932,</subfield>
+   <subfield code="e">composer.</subfield>
+  </datafield>
+ </record>
+ <record>
+  <leader>01137cam a2200301 a 4500</leader>
+  <datafield tag="245" ind1="1" ind2="0">
+   <subfield code="a">Two pieces /</subfield>
+   <subfield code="c">by Frank O&#39;Hara.</subfield>
+  </datafield>
+  <datafield tag="100" ind1="1" ind2=" ">
+   <subfield code="a">O&#39;Hara, Frank,</subfield>
+   <subfield code="d">1926-1966.</subfield>
+   <subfield code="0">http://id.loc.gov/authorities/names/n79042130</subfield>
+  </datafield>
+ </record>
+</collection>

data/lib/traject/debug_writer.rb

@@ -62,7 +62,7 @@ All records are assumed to have a unique id. You can set which field to look in
   def serialize(context)
     h       = context.output_hash
     rec_key = record_number(context)
-    lines   = h.keys.sort.map { |k| @format % [rec_key, k, h[k].join(' | ')] }
+    lines   = h.keys.sort.map { |k| @format % [rec_key, k, (h[k] || []).join(' | ')] }
     lines.push "\n"
     lines.join("\n")
   end

data/lib/traject/nokogiri_reader.rb

@@ -1,3 +1,5 @@
+require 'nokogiri'
+
 module Traject
   # A Trajet reader which reads XML, and yields zero to many Nokogiri::XML::Document
   # objects as source records in the traject pipeline.

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.5.0"
+  VERSION = "3.6.0"
 end

data/test/command_line_test.rb

@@ -22,15 +22,16 @@ describe "Shell out to command line" do
 
   it "can display version" do
     out, err, result = execute_with_args("-v")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
     assert_equal err, "traject version #{Traject::VERSION}\n"
-    assert result.success?
   end
 
   it "can display help text" do
     out, err, result = execute_with_args("-h")
 
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
     assert err.start_with?("traject [options] -c configuration.rb [-c config2.rb] file.mrc")
-    assert result.success?
   end
 
   it "handles bad argument" do
@@ -43,7 +44,7 @@ describe "Shell out to command line" do
   it "does basic dry run" do
     out, err, result = execute_with_args("--debug-mode -s one=two -s three=four -c test/test_support/demo_config.rb test/test_support/emptyish_record.marc")
 
-    assert result.success?
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
     assert_includes err, "executing with: `--debug-mode -s one=two -s three=four"
     assert_match /bib_1000165 +author_sort +Collection la/, out
   end

data/test/debug_writer_test.rb

@@ -73,6 +73,19 @@ describe 'Simple output' do
 
   end
 
+  it "deals ok with nil values" do
+    record_with_nil_value = {"id"=>["2710183"], "title"=>["Manufacturing consent : the political economy of the mass media /"], "xyz"=>nil}
+    @writer.put Traject::Indexer::Context.new(:output_hash => record_with_nil_value)
+    expected = [
+      "#{@id} id #{@id}",
+      "#{@id} title #{@title}",
+      "#{@id} xyz",
+      "\n"
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    @writer.close
+
+  end
 end
 
 

data/test/indexer/read_write_test.rb

@@ -7,7 +7,8 @@ memory_writer_class = Class.new do
       # store them in a class variable so we can test em later
       # Supress the warning message
       original_verbose, $VERBOSE = $VERBOSE, nil
-      @@last_writer_settings = @settings = settings
+      @settings = settings
+      self.class.store_last_writer_settings(@settings)
       # Activate warning messages again.
       $VERBOSE = original_verbose
       @settings["memory_writer.added"] = []
@@ -20,6 +21,16 @@ memory_writer_class = Class.new do
     def close
       @settings["memory_writer.closed"] = true
     end
+
+    private
+
+    def self.store_last_writer_settings(settings)
+      @last_writer_settings = settings
+    end
+
+    def self.last_writer_settings
+      @last_writer_settings
+    end
   end
 
 describe "Traject::Indexer#process" do
@@ -53,7 +64,7 @@ describe "Traject::Indexer#process" do
 
     # Grab the settings out of a class variable where we left em,
     # as a convenient place to store outcomes so we can test em.
-    writer_settings = memory_writer_class.class_variable_get("@@last_writer_settings")
+    writer_settings = memory_writer_class.last_writer_settings
 
     assert writer_settings["memory_writer.added"]
     assert_equal 30, writer_settings["memory_writer.added"].length
@@ -146,7 +157,7 @@ describe "Traject::Indexer#process" do
     it "parses and loads" do
       @indexer.process([@file1, @file2])
       # kinda ridic, yeah.
-      output_hashes = memory_writer_class.class_variable_get("@@last_writer_settings")["memory_writer.added"].collect(&:output_hash)
+      output_hashes = memory_writer_class.last_writer_settings["memory_writer.added"].collect(&:output_hash)
 
       assert_length 2, output_hashes
       assert output_hashes.all? { |hash| hash["title"].length > 0 }

data/traject.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
-  spec.add_dependency "http", ">= 3.0", "< 5" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
+  spec.add_dependency "http", ">= 3.0", "< 6" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.6.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-14 00:00:00.000000000 Z
+date: 2021-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -124,7 +124,7 @@ dependencies:
         version: '3.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -134,7 +134,7 @@ dependencies:
         version: '3.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: marc-fastxmlwriter
   requirement: !ruby/object:Gem::Requirement
@@ -257,6 +257,7 @@ files:
 - doc/programmatic_use.md
 - doc/settings.md
 - doc/xml.md
+- examples/marc/tiny.xml
 - lib/tasks/load_maps.rake
 - lib/traject.rb
 - lib/traject/array_writer.rb