traject 1.1.0 → 2.0.0.rc.1

data/test/test_helper.rb

@@ -47,9 +47,11 @@ def empty_record
   rec
 end
 
-# pretends to be a SolrJ HTTPServer-like thing, just kind of mocks it up
+# pretends to be a Solr HTTPServer-like thing, just kind of mocks it up
 # and records what happens and simulates errors in some cases.
 class MockSolrServer
+  class Exception < RuntimeError;end
+
   attr_accessor :things_added, :url, :committed, :parser, :shutted_down
 
   def initialize(url)
@@ -61,12 +63,12 @@ class MockSolrServer
   def add(thing)
     @add_mutex.synchronize do # easy peasy threadsafety for our mock
       if @url == "http://no.such.place"
-        raise org.apache.solr.client.solrj.SolrServerException.new("mock bad uri", java.io.IOException.new)
+        raise MockSolrServer::Exception.new("mock bad uri")
       end
 
       # simulate a multiple id error please
       if [thing].flatten.find {|doc| doc.getField("id").getValueCount() != 1}
-        raise org.apache.solr.client.solrj.SolrServerException.new("mock non-1 size of 'id'")
+        raise MockSolrServer::Exception.new("mock non-1 size of 'id'")
       else
         things_added << thing
       end

data/test/test_support/demo_config.rb

@@ -21,11 +21,6 @@ extend Traject::Macros::MarcFormats
 # config files as you like, `traject -c one.rb -c two.rb -c etc.rb`
 settings do
   provide "solr.url", "http://solr.somewhere.edu:8983/solr/corename"
-
-  # Only if you need to connect to a Solr 1.x:
-  provide "solrj_writer.parser_class_name", "XMLResponseParser"
-
-  provide "solrj_writer.commit_on_close", true
 end
 
 # Extract first 001, then supply code block to add "bib_" prefix to it

data/test/translation_map_test.rb

@@ -104,6 +104,15 @@ describe "TranslationMap" do
     assert_equal "output_value", map["input_value"]
   end
 
+  it "can be initialized with another map" do
+    map = Traject::TranslationMap.new({"alpha" => "one", "beta" => nil}, :default => "DEFAULT")
+
+    new_map = Traject::TranslationMap.new(map)
+
+    assert_equal map.to_hash, new_map.to_hash
+    assert_equal map.default, new_map.default
+  end
+
   it "respects __default__ literal" do
     map = Traject::TranslationMap.new("default_literal")
 

data/traject.gemspec

@@ -20,12 +20,25 @@ Gem::Specification.new do |spec|
   spec.extra_rdoc_files = spec.files.grep(%r{^doc/})
 
 
-  spec.add_dependency "marc", ">= 0.8.0"
-  spec.add_dependency "marc-marc4j", ">=0.1.1" # use and convert marc4j
-  spec.add_dependency "hashie", ">= 2.0.5", "< 2.1" # used for Indexer#settings
+  spec.add_dependency "concurrent-ruby", ">= 0.8.0"
+  spec.add_dependency "marc", "~> 1.0"
+
+  spec.add_dependency "hashie", "~> 3.1"            # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0"   # command line parsing
-  spec.add_dependency "yell" # logging
-  spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
+  spec.add_dependency "yell"                        # logging
+  spec.add_dependency "dot-properties", ">= 0.1.1"  # reading java style .properties
+  spec.add_dependency "httpclient", "~> 2.5"
+  
+  # If we're building the package under JRuby, add in the 
+  # jruby-only gems and specify the platform.
+  
+  if defined? JRUBY_VERSION
+    spec.platform = 'java'
+    spec.add_dependency "traject-marc4j_reader", "~> 1.0"
+  else
+    spec.platform = "ruby"
+  end
+  
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,155 +1,163 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0.rc.1
 platform: ruby
 authors:
 - Jonathan Rochkind
 - Bill Dueber
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-07 00:00:00.000000000 Z
+date: 2015-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
-  name: marc
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
 - !ruby/object:Gem::Dependency
+  name: marc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
-  name: marc-marc4j
-  prerelease: false
+        version: '1.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.5
-    - - <
-      - !ruby/object:Gem::Version
-        version: '2.1'
-  name: hashie
-  prerelease: false
+        version: '3.1'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.5
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.4.5
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '4.0'
-  name: slop
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.4.5
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
+  name: yell
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: yell
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: dot-properties
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.1
-  name: dot-properties
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.1
 - !ruby/object:Gem::Dependency
+  name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
-  name: bundler
+        version: '2.5'
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: minitest
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - none@nowhere.org
 executables:
@@ -162,9 +170,9 @@ extra_rdoc_files:
 - doc/other_commands.md
 - doc/settings.md
 files:
-- .gitignore
-- .travis.yml
-- .yardopts
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -179,7 +187,9 @@ files:
 - lib/tasks/load_maps.rake
 - lib/traject.rb
 - lib/traject/command_line.rb
+- lib/traject/csv_writer.rb
 - lib/traject/debug_writer.rb
+- lib/traject/delimited_writer.rb
 - lib/traject/indexer.rb
 - lib/traject/indexer/settings.rb
 - lib/traject/json_writer.rb
@@ -188,14 +198,13 @@ files:
 - lib/traject/macros/marc21.rb
 - lib/traject/macros/marc21_semantics.rb
 - lib/traject/macros/marc_format_classifier.rb
-- lib/traject/marc4j_reader.rb
 - lib/traject/marc_extractor.rb
 - lib/traject/marc_reader.rb
 - lib/traject/mock_reader.rb
 - lib/traject/ndj_reader.rb
 - lib/traject/null_writer.rb
 - lib/traject/qualified_const_get.rb
-- lib/traject/solrj_writer.rb
+- lib/traject/solr_json_writer.rb
 - lib/traject/thread_pool.rb
 - lib/traject/translation_map.rb
 - lib/traject/util.rb
@@ -208,6 +217,7 @@ files:
 - lib/translation_maps/marc_instruments.yaml
 - lib/translation_maps/marc_languages.yaml
 - test/debug_writer_test.rb
+- test/delimited_writer_test.rb
 - test/indexer/each_record_test.rb
 - test/indexer/macros_marc21_semantics_test.rb
 - test/indexer/macros_marc21_test.rb
@@ -216,11 +226,10 @@ files:
 - test/indexer/read_write_test.rb
 - test/indexer/settings_test.rb
 - test/indexer/to_field_test.rb
-- test/marc4j_reader_test.rb
 - test/marc_extractor_test.rb
 - test/marc_format_classifier_test.rb
 - test/marc_reader_test.rb
-- test/solrj_writer_test.rb
+- test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc
 - test/test_support/880_with_no_6.utf8.marc
@@ -263,51 +272,33 @@ files:
 - test/translation_maps/translate_array_test.yaml
 - test/translation_maps/yaml_map.yaml
 - traject.gemspec
-- vendor/solrj/README
-- vendor/solrj/build.xml
-- vendor/solrj/ivy.xml
-- vendor/solrj/lib/commons-codec-1.7.jar
-- vendor/solrj/lib/commons-io-2.1.jar
-- vendor/solrj/lib/httpclient-4.2.3.jar
-- vendor/solrj/lib/httpcore-4.2.2.jar
-- vendor/solrj/lib/httpmime-4.2.3.jar
-- vendor/solrj/lib/jcl-over-slf4j-1.6.6.jar
-- vendor/solrj/lib/jul-to-slf4j-1.6.6.jar
-- vendor/solrj/lib/log4j-1.2.16.jar
-- vendor/solrj/lib/noggit-0.5.jar
-- vendor/solrj/lib/slf4j-api-1.6.6.jar
-- vendor/solrj/lib/slf4j-log4j12-1.6.6.jar
-- vendor/solrj/lib/solr-solrj-4.3.1-javadoc.jar
-- vendor/solrj/lib/solr-solrj-4.3.1-sources.jar
-- vendor/solrj/lib/solr-solrj-4.3.1.jar
-- vendor/solrj/lib/wstx-asl-3.2.7.jar
-- vendor/solrj/lib/zookeeper-3.4.5.jar
 homepage: http://github.com/traject-project/traject
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
 specification_version: 4
 summary: Index MARC to Solr; or generally process source records to hash-like structures
 test_files:
 - test/debug_writer_test.rb
+- test/delimited_writer_test.rb
 - test/indexer/each_record_test.rb
 - test/indexer/macros_marc21_semantics_test.rb
 - test/indexer/macros_marc21_test.rb
@@ -316,11 +307,10 @@ test_files:
 - test/indexer/read_write_test.rb
 - test/indexer/settings_test.rb
 - test/indexer/to_field_test.rb
-- test/marc4j_reader_test.rb
 - test/marc_extractor_test.rb
 - test/marc_format_classifier_test.rb
 - test/marc_reader_test.rb
-- test/solrj_writer_test.rb
+- test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc
 - test/test_support/880_with_no_6.utf8.marc

data/lib/traject/marc4j_reader.rb

@@ -1,153 +0,0 @@
-require 'traject'
-require 'marc'
-require 'marc/marc4j'
-
-# `Traject::Marc4JReader` uses the marc4j java package to parse the MARC records
-# into standard ruby-marc MARC::Record objects. This reader may be faster than
-# Traject::MarcReader, especially for XML.
-#
-# Marc4JReader can read MARC ISO 2709 ("binary") or MARCXML. We use the Marc4J MarcPermissiveStreamReader
-# for reading binary, but sometimes in non-permissive mode, according to settings. We use the Marc4j MarcXmlReader
-# for reading xml. The actual code for dealing with Marc4J is in the separate
-# [marc-marc4j gem](https://github.com/billdueber/ruby-marc-marc4j).
-#
-# See also the pure ruby Traject::MarcReader as an alternative, if you need to read
-# marc-in-json, or if you don't need binary Marc8 support, it may in some cases
-# be faster.
-#
-# ## Settings
-#
-# * marc_source.type:     serialization type. default 'binary', also 'xml' (TODO: json/marc-in-json)
-#
-# * marc4j_reader.permissive:   default true, false to turn off permissive reading. Used as
-#                             value to 'permissive' arg of MarcPermissiveStreamReader constructor.
-#                             Only used for 'binary'
-#
-# * marc_source.encoding: Only used for 'binary', otherwise always UTF-8.
-#         String of the values MarcPermissiveStreamReader accepts:
-#         * BESTGUESS  (default: not entirely clear what Marc4J does with this)
-#         * ISO-8859-1 (also accepted: ISO8859_1)
-#         * UTF-8
-#         * MARC-8 (also accepted: MARC8)
-#         Default 'BESTGUESS', but HIGHLY recommend setting
-#         to avoid some Marc4J unpredictability, Marc4J "BESTGUESS" can be unpredictable
-#         in a variety of ways. 
-#         (will ALWAYS be transcoded to UTF-8 on the way out. We insist.)
-#
-# * marc4j_reader.jar_dir: Path to a directory containing Marc4J jar file to use. All .jar's in dir will
-#                          be loaded. If unset, uses marc4j.jar bundled with traject.
-#
-# * marc4j_reader.keep_marc4j: Keeps the original marc4j record accessible from
-#   the eventual ruby-marc record via record#original_marc4j. Intended for
-#   those that have legacy java code for which a marc4j object is needed. .
-#
-#
-# ## Example
-#
-# In a configuration file:
-#
-#     require 'traject/marc4j_reader
-#     settings do
-#       provide "reader_class_name", "Traject::Marc4JReader"
-#
-#       #for MarcXML:
-#       # provide "marc_source.type", "xml"
-#
-#       # Or instead for binary:
-#       provide "marc4j_reader.permissive", true
-#       provide "marc_source.encoding", "MARC8"
-#     end
-class Traject::Marc4JReader
-  include Enumerable
-
-  attr_reader :settings, :input_stream
-
-  def initialize(input_stream, settings)
-    @settings     = Traject::Indexer::Settings.new settings
-    @input_stream = input_stream
-
-    if @settings['marc4j_reader.keep_marc4j'] &&
-      ! (MARC::Record.instance_methods.include?(:original_marc4j) &&
-         MARC::Record.instance_methods.include?(:"original_marc4j="))
-      MARC::Record.class_eval('attr_accessor :original_marc4j')
-    end
-
-    # Creating a converter will do the following:
-    #  - nothing, if it detects that the marc4j jar is already loaded
-    #  - load all the .jar files in settings['marc4j_reader.jar_dir'] if set
-    #  - load the marc4j jar file bundled with MARC::MARC4J otherwise
-
-    @converter = MARC::MARC4J.new(:jardir => settings['marc4j_reader.jar_dir'], :logger => logger)
-
-    # Convenience
-    java_import org.marc4j.MarcPermissiveStreamReader
-    java_import org.marc4j.MarcXmlReader
-
-  end
-
-
-  def internal_reader
-    @internal_reader ||= create_marc_reader!
-  end
-
-  def input_type
-    # maybe later add some guessing somehow
-    settings["marc_source.type"]
-  end
-
-  def specified_source_encoding
-    #settings["marc4j_reader.source_encoding"]
-    enc = settings["marc_source.encoding"]
-
-    # one is standard for ruby and we want to support,
-    # the other is used by Marc4J and we have to pass it to Marc4J
-    enc = "ISO8859_1" if enc == "ISO-8859-1"
-
-    # default
-    enc = "BESTGUESS" if enc.nil? || enc.empty?
-
-    return enc
-  end
-
-  def create_marc_reader!
-    case input_type
-    when "binary"
-      permissive = settings["marc4j_reader.permissive"].to_s == "true"
-
-      # #to_inputstream turns our ruby IO into a Java InputStream
-      # third arg means 'convert to UTF-8, yes'
-      MarcPermissiveStreamReader.new(input_stream.to_inputstream, permissive, true, specified_source_encoding)
-    when "xml"
-      MarcXmlReader.new(input_stream.to_inputstream)
-    else
-      raise IllegalArgument.new("Unrecgonized marc_source.type: #{input_type}")
-    end
-  end
-
-  def each
-    while (internal_reader.hasNext)
-      begin
-        marc4j = internal_reader.next
-        rubymarc = @converter.marc4j_to_rubymarc(marc4j)
-        if @settings['marc4j_reader.keep_marc4j']
-          rubymarc.original_marc4j = marc4j
-        end
-      rescue Exception =>e
-        msg = "MARC4JReader: Error reading MARC, fatal, re-raising"
-        if marc4j
-          msg += "\n    001 id: #{marc4j.getControlNumber}"
-        end
-        msg += "\n    #{Traject::Util.exception_to_log_message(e)}"
-        logger.fatal msg
-        raise e
-      end
-
-      yield rubymarc
-    end
-  end
-
-  def logger
-    @logger ||= (settings[:logger] || Yell.new(STDERR, :level => "gt.fatal")) # null logger)
-  end
-
-end