traject 2.0.0.rc.1 → 2.0.0.rc.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e875abe713a3200de4fb424c9570a3b89d5ddc0
-  data.tar.gz: cbb7b0f4fd9bb293af55afff48b52f92ce2b4dfb
+  metadata.gz: 6fbaf2787d85b83ff8df29436475a8366a6f89d3
+  data.tar.gz: 5487c8f6614e6875131ee821fd5ad85f45eaad06
 SHA512:
-  metadata.gz: b7f911f43784275b0a7782e642788bb57a492c1660b9e3a461bf9b03c96882b887dd62e18f385ee9ffb3488c60afda44fb77ea9fc619cd4f3061c471b5bc7227
-  data.tar.gz: 105737429ce7778ae57a3182671fa9c41f72d17579b82f80b38a8ec373b634c1d22394e17c404e23961b5d2f557442e56d69700aaad3091e950d36aa2dc050c5
+  metadata.gz: 20b254aedcae88a4f0392900677373b86f337b0ccb04e4ef27841c90621620a828e7572769a50049f402e30a81156a5f9988a2c144d5db513bcaa6f70d938af2
+  data.tar.gz: f8f9bd42c8c11e375d45a33e58341f54c5fbc8060d7eeff48abd0096615b993efb11ce78337ab131d6137f93ec0be039ac94f89bc611a4075752943ae114e27e

data/README.md

@@ -31,11 +31,10 @@ that can combine to deal with any of your local needs.
 
 ## Installation
 
-Traject runs under MRI ruby (1.9 through 2.2), jruby 1.7.x, or rubinius. 
+Traject runs under jruby (1.7.x or higher), MRI ruby (1.9.3 or higher), or probably any other ruby platform.
 
-For high-volume indexing in production, traject performs **much** better when run with **JRuby** (ruby on the JVM). 
-Standard MRI ruby can't use multiple CPU cores at once, but on JRuby traject can use
-multiple cores for much better performance. 
+**Traject runs much faster on JRuby** where it can use multi-core parallelism, and the Java
+Marc4J marc reader. If performance is a concern, you should run traject on JRuby. 
 
 Some options for installing a ruby other than your system-provided one are [chruby](https://github.com/postmodern/chruby) and [ruby-install](https://github.com/postmodern/ruby-install#readme).
 
@@ -150,7 +149,7 @@ to matched specifications, but you can turn that off, or extract *only* correspo
     to_field "title_vernacular", extract_marc("245abc", :alternate_script => :only)
 ~~~
 
-By default, specifications with multiple subfields (like "240abc") will produce one single string of output for each matching field. Specifications with single subfields (like "020a") will split subfields and produce an output string for each matching subfield.
+By default, specifications with multiple subfields (like "240abc") will produce one single string of output per field (for each '240'), with the concatenation of each matched subfield. Specifications with single subfields (like "020a") will split subfields and produce an output string for each matching subfield.
 
 For the syntax and complete possibilities of the specification
 string argument to extract_marc, see docs at the [MarcExtractor class](./lib/traject/marc_extractor.rb) ([rdoc](http://rdoc.info/gems/traject/Traject/MarcExtractor)).
@@ -427,11 +426,6 @@ Run tests with `rake test` or just `rake`.  Tests are written using Minitest (pl
 list the tests -- but generally prefer unit-style "assert_*" methods
 to make actual assertions, for clarity.
 
-Some tests need to run against a solr instance. Currently no solr
-instance is baked in.  You can provide your own solr instance to test against and set shell ENV variable
-"solr_url", and the tests will use it. Otherwise, tests will
-use a mocked up Solr instance.
-
 To make a pull request, please make a feature branch *created from the master branch*, not from an existing feature branch. (If you need to do a feature branch dependent on an existing not-yet merged feature branch... discuss
 this with other developers first!)
 
@@ -442,6 +436,9 @@ and/or extra files in ./docs -- as appropriate for what needs to be docs.
 online api docs has a `--markup markdown` specified -- inline class/method docs are in markdown, not rdoc.
 
 Bundler rake tasks included for gem releases: `rake release`
+* Every traject release needs to be done once when running MRI, and switch to JRuby
+and do the same release again. The JRuby release is identical but for including
+a gemspec dependency on the Marc4JReader gem. 
 
 ## TODO
 

data/lib/traject/indexer.rb

@@ -324,13 +324,9 @@ class Traject::Indexer
 
 
     processing_threads = settings["processing_thread_pool"].to_i
-    if processing_threads > 0 and !(defined? JRuby)
-      processing_threads = 0
-      logger.warn "Processing threads set to 0 because we're not running under JRuby"
-    end
     thread_pool = Traject::ThreadPool.new(processing_threads)
 
-    logger.info "   Indexer with reader: #{reader.class.name} and writer: #{writer.class.name}"
+    logger.info "   Indexer with #{processing_threads} processing threads, reader: #{reader.class.name} and writer: #{writer.class.name}"
 
     log_batch_size = settings["log.batch_size"] && settings["log.batch_size"].to_i
 

data/lib/traject/macros/marc21.rb

@@ -3,6 +3,7 @@ require 'traject/translation_map'
 require 'traject/util'
 require 'base64'
 require 'json'
+require 'marc/fastxmlwriter'
 
 module Traject::Macros
   # Some of these may be generic for any MARC, but we haven't done
@@ -140,9 +141,7 @@ module Traject::Macros
           binary = Base64.encode64(binary) if binary_escape
           accumulator << binary
         when "xml"
-          # ruby-marc #to_xml returns a REXML object at time of this writing, bah!@
-          # call #to_s on it. Hopefully that'll be forward compatible.
-          accumulator << record.to_xml.to_s
+          accumulator << MARC::FastXMLWriter.encode(record)
         when "json"
           accumulator << JSON.dump(record.to_hash)
         end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "2.0.0.rc.1"
+  VERSION = "2.0.0.rc.2"
 end

data/test/indexer/macros_marc21_test.rb

@@ -4,7 +4,7 @@ require 'traject/indexer'
 require 'traject/macros/marc21'
 
 require 'json'
-require 'marc/record'
+require 'marc'
 
 # See also marc_extractor_test.rb for more detailed tests on marc extraction,
 # this is just a basic test to make sure our macro works passing through to there
@@ -127,7 +127,8 @@ describe "Traject::Macros::Marc21" do
 
       assert_length 1, output["marc_record"]
       assert_kind_of String, output["marc_record"].first
-      assert output["marc_record"].first.start_with?("<record xmlns='http://www.loc.gov/MARC21/slim'>"), "looks like serialized MarcXML"
+      roundtrip_record = MARC::XMLReader.new(StringIO.new(output["marc_record"].first)).first
+      assert_equal @record, roundtrip_record
     end
 
     it "serializes binary UUEncoded" do

data/traject.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "yell"                        # logging
   spec.add_dependency "dot-properties", ">= 0.1.1"  # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
+  spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   
   # If we're building the package under JRuby, add in the 
   # jruby-only gems and specify the platform.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc.1
+  version: 2.0.0.rc.2
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-11 00:00:00.000000000 Z
+date: 2015-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -115,6 +115,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: marc-fastxmlwriter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement