traject 3.4.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c30572335810dc620f9a169df6f8f374512d3c472ea34bc03068106959fd1463
-  data.tar.gz: 3181c37e41e80416487d730e1983bc647daf480a3a308db30e294c7587adc644
+  metadata.gz: 01ca968682bb3fc2a8313131ef6344bfc9e5418b767b2900c3d799caa356d016
+  data.tar.gz: a3fd6c9a3bec88c6ba592500ea170357b059533f28c5ba3fb2fe72de39702a2a
 SHA512:
-  metadata.gz: 83b73a10113e75106a0fb7af9bec79802d2e3f5c8f3e07742f33a52642a9441c20769072f8ea5bd532011b7d172db6ca007121d6874f705008e1a5a511ca1ff8
-  data.tar.gz: d9c53588e8adbd76764c20012baf702591276d84c2cd64ed0bb0d5b742699607a2287d30a2fb4f70c1bf6f8a7338d716d989b208c8983c2a87eeddbd6d96dd3d
+  metadata.gz: 93547927e90b7947588c1983bd37de4651722bebaff7aaad2d3965ec46eed8c647971f1ce093beb86a5c47c2efa999516d00fb6956682c98913cd54ea5a1a2b8
+  data.tar.gz: 128ea6e2517711f2324541215f814430f963b0042ae321ebc3f29646398990dc09d2a307acc32fb89a70142d74763fdfee6a61d220043622b5b8b11fbcd645d8

data/.github/workflows/ruby.yml

@@ -0,0 +1,26 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: ['**']
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7', 'jruby-9.1', 'jruby-9.2' ]
+    name: Ruby ${{ matrix.ruby }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Install dependencies
+      run: bundle install --jobs 4 --retry 3
+    - name: Run tests
+      run: bundle exec rake

data/CHANGES.md

@@ -6,6 +6,17 @@
 
 *
 
+*
+
+## 3.5.0
+
+* `traject -v` and `traject -h` correctly return 0 exit code indicating success.
+
+* upgrade to slop gem  4.x, which carries with it a slightly different format of human-readable command-line arg errors, should be otherwise invisible.
+
+* the SolrJsonWriter now supports HTTP basic auth credentials embedded in `solr.url` or `solr.update_url`, eg `http://user:pass@example.org/solr` https://github.com/traject/traject/pull/262
+
+
 ## 3.4.0
 
 * XML-mode `extract_xpath` now supports extracting attribute values with xpath @attr syntax.

data/README.md

@@ -9,7 +9,7 @@ Traject can also be generalized to a set of tools for getting structured data fr
 **Traject is stable, mature software, that is already being used in production by its authors and several other institutions.**
 
 [![Gem Version](https://badge.fury.io/rb/traject.png)](http://badge.fury.io/rb/traject)
-[![Build Status](https://travis-ci.org/traject/traject.png)](https://travis-ci.org/traject/traject)
+[![CI Status](https://github.com/traject/traject/workflows/CI/badge.svg?branch=master)](https://github.com/traject/traject/actions?query=workflow%3ACI+branch%3Amaster)
 
 
 ## Background/Goals

data/doc/settings.md

@@ -83,7 +83,8 @@ settings are applied first of all. It's recommended you use `provide`.
 ### Writing to solr
 
 * `json_writer.pretty_print`: used by the JsonWriter, if set to true, will output pretty printed json (with added whitespace) for easier human readability. Default false.
-* `solr.url`: URL to connect to a solr instance for indexing, eg http://example.org:8983/solr . Command-line short-cut `-u`.
+
+* `solr.url`: URL to connect to a solr instance for indexing, eg http://example.org:8983/solr . Command-line short-cut `-u`. (Can include embedded HTTP basic auth as eg `http://user:pass@example.org/solr`)
 
 * `solr.version`: Set to eg "1.4.0", "4.3.0"; currently un-used, but in the future will control some default settings, and/or sanity check and warn you if you're doing something that might not work with that version of solr. Set now for help in the future.
 
@@ -93,7 +94,8 @@ settings are applied first of all. It's recommended you use `provide`.
 
 * `solr_writer.thread_pool`: defaults to 1 (single bg thread). A thread pool is used for submitting docs to solr. Set to 0 or nil to disable threading. Set to 1, there will still be a single bg thread doing the adds. May make sense to set higher than number of cores on your indexing machine, as these threads will mostly be waiting on Solr. Speed/capacity of your solr might be more relevant. Note that processing_thread_pool threads can end up submitting to solr too, if solr_json_writer.thread_pool is full.
 
-* `solr_writer.basic_auth_user`, `solr_writer.basic_auth_password`: Not set by default but when both are set the default writer is configured with basic auth.
+* `solr_writer.basic_auth_user`, `solr_writer.basic_auth_password`: Not set by default but when both are set the default writer is configured with basic auth. You can also just embed basic
+auth credentials in `solr.url` using standard URI syntax.
 
 
 ### Dealing with MARC data

data/lib/traject/command_line.rb

@@ -29,10 +29,10 @@ module Traject
       self.console = $stderr
 
       self.orig_argv      = argv.dup
-      self.remaining_argv = argv
 
-      self.slop    = create_slop!
-      self.options = parse_options(self.remaining_argv)
+      self.slop    = create_slop!(argv)
+      self.options = self.slop
+      self.remaining_argv = self.slop.arguments
     end
 
     # Returns true on success or false on failure; may also raise exceptions;
@@ -40,11 +40,11 @@ module Traject
     def execute
       if options[:version]
         self.console.puts "traject version #{Traject::VERSION}"
-        return
+        return true
       end
       if options[:help]
-        self.console.puts slop.help
-        return
+        self.console.puts slop.to_s
+        return true
       end
 
 
@@ -179,11 +179,11 @@ module Traject
     end
 
     def arg_check!
-      if options[:command] == "process" && (options[:conf].nil? || options[:conf].length == 0)
+      if options[:command] == "process" && (!options[:conf] || options[:conf].length == 0)
         self.console.puts "Error: Missing required configuration file"
         self.console.puts "Exiting..."
         self.console.puts
-        self.console.puts self.slop.help
+        self.console.puts self.slop.to_s
         exit 2
       end
     end
@@ -234,28 +234,36 @@ module Traject
     end
 
 
-    def create_slop!
-      return Slop.new(:strict => true) do
-        banner "traject [options] -c configuration.rb [-c config2.rb] file.mrc"
+    def create_slop!(argv)
+      options = Slop::Options.new do |o|
+        o.banner = "traject [options] -c configuration.rb [-c config2.rb] file.mrc"
 
-        on 'v', 'version', "print version information to stderr"
-        on 'd', 'debug', "Include debug log, -s log.level=debug"
-        on 'h', 'help', "print usage information to stderr"
-        on 'c', 'conf', 'configuration file path (repeatable)', :argument => true, :as => Array
-        on :i, 'indexer', "Traject indexer class name or shortcut", :argument => true, default: "marc"
-        on :s, :setting, "settings: `-s key=value` (repeatable)", :argument => true, :as => Array
-        on :r, :reader, "Set reader class, shortcut for -s reader_class_name=", :argument => true
-        on :o, "output_file", "output file for Writer classes that write to files", :argument => true
-        on :w, :writer, "Set writer class, shortcut for -s writer_class_name=", :argument => true
-        on :u, :solr, "Set solr url, shortcut for -s solr.url=", :argument => true
-        on :t, :marc_type, "xml, json or binary. shortcut for -s marc_source.type=", :argument => true
-        on :I, "load_path", "append paths to ruby $LOAD_PATH", :argument => true, :as => Array, :delimiter => ":"
+        o.on '-v', '--version', "print version information to stderr"
+        o.on '-d', '--debug', "Include debug log, -s log.level=debug"
+        o.on '-h', '--help', "print usage information to stderr"
+        o.array '-c', '--conf', 'configuration file path (repeatable)', :delimiter => nil
+        o.string "-i", '--indexer', "Traject indexer class name or shortcut", :default => "marc"
+        o.array "-s", "--setting", "settings: `-s key=value` (repeatable)", :delimiter => nil
+        o.string "-r", "--reader", "Set reader class, shortcut for -s reader_class_name="
+        o.string "-o", "--output_file", "output file for Writer classes that write to files"
+        o.string "-w", "--writer", "Set writer class, shortcut for -s writer_class_name="
+        o.string "-u", "--solr", "Set solr url, shortcut for -s solr.url="
+        o.string "-t", "--marc_type", "xml, json or binary. shortcut for -s marc_source.type="
+        o.array "-I", "--load_path", "append paths to ruby $LOAD_PATH", :delimiter => ":"
 
-        on :x, "command", "alternate traject command: process (default); marcout; commit", :argument => true, :default => "process"
+        o.string "-x", "--command", "alternate traject command: process (default); marcout; commit", :default => "process"
 
-        on "stdin", "read input from stdin"
-        on "debug-mode", "debug logging, single threaded, output human readable hashes"
+        o.on "--stdin", "read input from stdin"
+        o.on "--debug-mode", "debug logging, single threaded, output human readable hashes"
       end
+
+      options.parse(argv)
+    rescue Slop::Error => e
+      self.console.puts "Error: #{e.message}"
+      self.console.puts "Exiting..."
+      self.console.puts
+      self.console.puts options.to_s
+      exit 1
     end
 
     def initialize_indexer!
@@ -267,22 +275,5 @@ module Traject
 
       return indexer
     end
-
-    def parse_options(argv)
-
-      begin
-        self.slop.parse!(argv)
-      rescue Slop::Error => e
-        self.console.puts "Error: #{e.message}"
-        self.console.puts "Exiting..."
-        self.console.puts
-        self.console.puts slop.help
-        exit 1
-      end
-
-      return self.slop.to_hash
-    end
-
-
   end
 end

data/lib/traject/macros/marc21.rb

@@ -15,7 +15,7 @@ module Traject::Macros
     # field/substring specification.
     #
     # First argument is a string spec suitable for the MarcExtractor, see
-    # MarcExtractor::parse_string_spec.
+    # Traject::MarcExtractor::Spec.
     #
     # Second arg is optional options, including options valid on MarcExtractor.new,
     # and others. By default, will de-duplicate results, but see :allow_duplicates

data/lib/traject/marc_extractor.rb

@@ -2,9 +2,9 @@ require 'traject/marc_extractor_spec'
 
 module Traject
   # MarcExtractor is a class for extracting lists of strings from a MARC::Record,
-  # according to specifications. See #parse_string_spec for description of string
-  # string arguments used to specify extraction. See #initialize for options
-  # that can be set controlling extraction.
+  # according to specifications. See Traject::MarcExtractor::Spec for description
+  # of string string arguments used to specify extraction. See #initialize for
+  # options that can be set controlling extraction.
   #
   # Examples:
   #

data/lib/traject/solr_json_writer.rb

@@ -41,10 +41,12 @@ require 'concurrent' # for atomic_fixnum
 #
 # ## Relevant settings
 #
-# * solr.url (optional if solr.update_url is set) The URL to the solr core to index into
+# * solr.url (optional if solr.update_url is set) The URL to the solr core to index into.
+#   (Can include embedded HTTP basic auth as eg `http://user:pass@host/solr`)
 #
 # * solr.update_url: The actual update url. If unset, we'll first see if
-#   "#{solr.url}/update/json" exists, and if not use "#{solr.url}/update"
+#   "#{solr.url}/update/json" exists, and if not use "#{solr.url}/update". (Can include
+#   embedded HTTP basic auth as eg `http://user:pass@host/solr)
 #
 # * solr_writer.batch_size: How big a batch to send to solr. Default is 100.
 #   My tests indicate that this setting doesn't change overall index speed by a ton.
@@ -101,12 +103,17 @@ class Traject::SolrJsonWriter
   def initialize(argSettings)
     @settings = Traject::Indexer::Settings.new(argSettings)
 
+
     # Set max errors
     @max_skipped = (@settings['solr_writer.max_skipped'] || DEFAULT_MAX_SKIPPED).to_i
     if @max_skipped < 0
       @max_skipped = nil
     end
 
+
+    # Figure out where to send updates, and if with basic auth
+    @solr_update_url, basic_auth_user, basic_auth_password = self.determine_solr_update_url
+
     @http_client = if @settings["solr_json_writer.http_client"]
       @settings["solr_json_writer.http_client"]
     else
@@ -115,9 +122,8 @@ class Traject::SolrJsonWriter
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
 
-      if @settings["solr_writer.basic_auth_user"] &&
-          @settings["solr_writer.basic_auth_password"]
-        client.set_auth(@settings["solr.url"], @settings["solr_writer.basic_auth_user"], @settings["solr_writer.basic_auth_password"])
+      if basic_auth_user || basic_auth_password
+        client.set_auth(@solr_update_url, basic_auth_user, basic_auth_password)
       end
 
       client
@@ -143,13 +149,11 @@ class Traject::SolrJsonWriter
     # this the new default writer.
     @commit_on_close = (settings["solr_writer.commit_on_close"] || settings["solrj_writer.commit_on_close"]).to_s == "true"
 
-    # Figure out where to send updates
-    @solr_update_url = self.determine_solr_update_url
 
     @solr_update_args = settings["solr_writer.solr_update_args"]
     @commit_solr_update_args = settings["solr_writer.commit_solr_update_args"]
 
-    logger.info("   #{self.class.name} writing to '#{@solr_update_url}' in batches of #{@batch_size} with #{@thread_pool_size} bg threads")
+    logger.info("   #{self.class.name} writing to '#{@solr_update_url}' #{"(with HTTP basic auth)" if basic_auth_user || basic_auth_password}in batches of #{@batch_size} with #{@thread_pool_size} bg threads")
   end
 
 
@@ -368,13 +372,27 @@ class Traject::SolrJsonWriter
   end
 
 
-  # Relatively complex logic to determine if we have a valid URL and what it is
+  # Relatively complex logic to determine if we have a valid URL and what it is,
+  # and if we have basic_auth info
+  #
+  # Empties out user and password embedded in URI returned, to help avoid logging it.
+  #
+  # @returns [update_url, basic_auth_user, basic_auth_password]
   def determine_solr_update_url
-    if settings['solr.update_url']
+    url = if settings['solr.update_url']
       check_solr_update_url(settings['solr.update_url'])
     else
       derive_solr_update_url_from_solr_url(settings['solr.url'])
     end
+
+    parsed_uri                            = URI.parse(url)
+    user_from_uri, password_from_uri      = parsed_uri.user, parsed_uri.password
+    parsed_uri.user, parsed_uri.password  = nil, nil
+
+    basic_auth_user     = @settings["solr_writer.basic_auth_user"] || user_from_uri
+    basic_auth_password = @settings["solr_writer.basic_auth_password"] || password_from_uri
+
+    return [parsed_uri.to_s, basic_auth_user, basic_auth_password]
   end
 
 

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.4.0"
+  VERSION = "3.5.0"
 end

data/test/command_line_test.rb

@@ -0,0 +1,51 @@
+# we mostly unit test with a Traject::Indexer itself and lower-level, but
+# we need at least some basic top-level integration actually command line tests,
+# this is a start, we can add more.
+#
+# Should we be testing Traject::CommandLine as an object instead of/in addition to
+# actually testing shell-out to command line call? Maybe.
+
+require 'test_helper'
+
+describe "Shell out to command line" do
+  # just encapsuluate using the minitest capture helper, but also
+  # getting and returning exit code
+  #
+  #     out, err, result = execute_with_args("-c configuration")
+  def execute_with_args(args)
+    out, err = capture_subprocess_io do
+      system("./bin/traject #{args}")
+    end
+
+    return out, err, $?
+  end
+
+  it "can display version" do
+    out, err, result = execute_with_args("-v")
+    assert_equal err, "traject version #{Traject::VERSION}\n"
+    assert result.success?
+  end
+
+  it "can display help text" do
+    out, err, result = execute_with_args("-h")
+
+    assert err.start_with?("traject [options] -c configuration.rb [-c config2.rb] file.mrc")
+    assert result.success?
+  end
+
+  it "handles bad argument" do
+    out, err, result = execute_with_args("--no-such-arg")
+    refute result.success?
+
+    assert err.start_with?("Error: unknown option `--no-such-arg'\nExiting...\n")
+  end
+
+  it "does basic dry run" do
+    out, err, result = execute_with_args("--debug-mode -s one=two -s three=four -c test/test_support/demo_config.rb test/test_support/emptyish_record.marc")
+
+    assert result.success?
+    assert_includes err, "executing with: `--debug-mode -s one=two -s three=four"
+    assert_match /bib_1000165 +author_sort +Collection la/, out
+  end
+end
+

data/test/solr_json_writer_test.rb

@@ -178,17 +178,52 @@ describe "Traject::SolrJsonWriter" do
     assert(auth.empty?)
   end
 
-  it "allows basic authentication setup" do
-    settings = {
-      "solr.url" => "http://example.com/solr/foo",
-      "solr_writer.basic_auth_user" => "foo",
-      "solr_writer.basic_auth_password" => "bar",
-    }
+  describe "HTTP basic auth" do
 
-    writer = Traject::SolrJsonWriter.new(settings)
-    auth = writer.instance_variable_get("@http_client")
-      .www_auth.basic_auth.instance_variable_get("@auth")
-    assert(!auth.empty?)
+    it "supports basic authentication settings" do
+      settings = {
+        "solr.url" => "http://example.com/solr/foo",
+        "solr_writer.basic_auth_user" => "foo",
+        "solr_writer.basic_auth_password" => "bar",
+      }
+
+      # testing with some internal implementation of HTTPClient sorry
+
+      writer = Traject::SolrJsonWriter.new(settings)
+
+      auth = writer.instance_variable_get("@http_client")
+        .www_auth.basic_auth.instance_variable_get("@auth")
+      assert(!auth.empty?)
+      assert_equal(auth.values.first, Base64.encode64("foo:bar").chomp)
+    end
+
+    it "supports basic auth from solr.url" do
+      settings = {
+        "solr.url" => "http://foo:bar@example.com/solr/foo",
+      }
+
+      # testing with some internal implementation of HTTPClient sorry
+
+      writer = Traject::SolrJsonWriter.new(settings)
+      auth = writer.instance_variable_get("@http_client")
+        .www_auth.basic_auth.instance_variable_get("@auth")
+      assert(!auth.empty?)
+      assert_equal(auth.values.first, Base64.encode64("foo:bar").chomp)
+    end
+
+    it "does not log basic auth from solr.url" do
+      string_io = StringIO.new
+      settings = {
+        "solr.url" => "http://secret_username:secret_password@example.com/solr/foo",
+        "logger"   => Logger.new(string_io)
+      }
+
+
+      writer = Traject::SolrJsonWriter.new(settings)
+
+      refute_includes string_io.string, "secret_username:secret_password"
+      assert_includes string_io.string, "(with HTTP basic auth)"
+    end
   end
 
   describe "commit" do

data/traject.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "marc", "~> 1.0"
 
   spec.add_dependency "hashie", ">= 3.1", "< 5" # used for Indexer#settings
-  spec.add_dependency "slop", ">= 3.4.5", "< 4.0" # command line parsing
+  spec.add_dependency "slop", "~> 4.0" # command line parsing
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-07 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -63,20 +63,14 @@ dependencies:
   name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.5
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.5
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
@@ -246,8 +240,8 @@ extra_rdoc_files:
 - doc/settings.md
 - doc/xml.md
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
-- ".travis.yml"
 - ".yardopts"
 - CHANGES.md
 - Gemfile
@@ -307,6 +301,7 @@ files:
 - lib/translation_maps/marc_geographic.yaml
 - lib/translation_maps/marc_instruments.yaml
 - lib/translation_maps/marc_languages.yaml
+- test/command_line_test.rb
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
 - test/experimental_nokogiri_streaming_reader_test.rb
@@ -411,6 +406,7 @@ specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
   system, focused on library-archives-museums input, and indexing to Solr as output.
 test_files:
+- test/command_line_test.rb
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
 - test/experimental_nokogiri_streaming_reader_test.rb

data/.travis.yml

@@ -1,17 +0,0 @@
-language: ruby
-cache: bundler
-# we don't really need `sudo: true`, but for some reason travis docker-based systems are unreliable
-# at downloading jruby, and
-sudo: true
-rvm:
-  - 2.4.4
-  - 2.5.1
-  - 2.6.1
-  - 2.7.0
-# avoid having travis install jdk on MRI builds where we don't need it.
-matrix:
-  include:
-    - jdk: openjdk8
-      rvm: jruby-9.1.17.0
-    - jdk: openjdk8
-      rvm: jruby-9.2.6.0