traject 0.0.2 → 0.9.1

data/test/solrj_writer_test.rb

@@ -2,6 +2,50 @@ require 'test_helper'
 
 require 'traject/solrj_writer'
 
+# It's crazy hard to test this effectively, especially under threading.
+# we do our best to test decently, and keep the tests readable,
+# but some things aren't quite reliable under threading, sorry.
+
+# create's a solrj_writer, maybe with MockSolrServer, maybe
+# with a real one. With settings in @settings, set or change
+# in before blocks
+#
+# writer left in @writer, with maybe mock solr server in @mock
+def create_solrj_writer
+  @writer = Traject::SolrJWriter.new(@settings)
+
+  if @settings["solrj_writer.server_class_name"] == "MockSolrServer"
+    # so we can test it later
+    @mock = @writer.solr_server
+  end
+end
+
+def context_with(hash)
+  Traject::Indexer::Context.new(:output_hash => hash)
+end
+
+
+# Some tests we need to run multiple ties in multiple batch/thread scenarios,
+# we DRY them up by creating a method to add the tests in different describe blocks
+def test_handles_errors
+  it "errors but does not raise on multiple ID's" do
+    @writer.put context_with("id" => ["one", "two"])
+    @writer.close
+    assert_equal 1, @writer.skipped_record_count, "counts skipped record"
+  end
+
+  it "errors and raises on connection error" do
+    @settings.merge!("solr.url" => "http://no.such.place")
+    create_solrj_writer
+    assert_raises org.apache.solr.client.solrj.SolrServerException do
+      @writer.put context_with("id" => ["one"])
+      # in batch and/or thread scenarios, sometimes no exception raised until close
+      @writer.close
+    end
+  end
+end
+
+$stderr.puts "\n======\nWARNING: Testing SolrJWriter with mock instance, set ENV 'solr_url' to test against real solr\n======\n\n" unless ENV["solr_url"]
 # WARNING. The SolrJWriter talks to a running Solr server.
 #
 # set ENV['solr_url'] to run tests against a real solr server
@@ -11,34 +55,45 @@ require 'traject/solrj_writer'
 #
 # This is pretty limited test right now.
 describe "Traject::SolrJWriter" do
+  before do
+    @settings = {
+      # Use XMLResponseParser just to test, and so it will work
+      # with a solr 1.4 test server
+      "solrj_writer.parser_class_name" => "XMLResponseParser",
+      "solrj_writer.commit_on_close" => "false", # real solr is way too slow if we always have it commit on close
+      "solrj_writer.batch_size" => nil
+    }
+
+    if ENV["solr_url"]
+      @settings["solr.url"] = ENV["solr_url"]
+    else
+      @settings["solr.url"] = "http://example.org/solr"
+      @settings["solrj_writer.server_class_name"] = "MockSolrServer"
+    end
+  end
 
   it "raises on missing url" do
     assert_raises(ArgumentError) { Traject::SolrJWriter.new }
     assert_raises(ArgumentError) { Traject::SolrJWriter.new("solr.url" => nil) }
   end
 
-  describe "with good setup" do
-    before do
-      @settings = {
-        # Use XMLResponseParser just to test, and so it will work
-        # with a solr 1.4 test server
-        "solrj_writer.parser_class_name" => "XMLResponseParser",
-        "solrj_writer.commit_on_close" => "true"
-      }
-
-      if ENV["solr_url"]
-        @settings["solr.url"] = ENV["solr_url"]
-      else
-        $stderr.puts "WARNING: Testing SolrJWriter with mock instance"
-        @settings["solr.url"] = "http://example.org/solr"
-        @settings["solrj_writer.server_class_name"] = "MockSolrServer"
-      end
+  it "raises on malformed URL" do
+    assert_raises(ArgumentError) { Traject::SolrJWriter.new("solr.url" => "") }
+    assert_raises(ArgumentError) { Traject::SolrJWriter.new("solr.url" => "adfadf") }
+  end
+
+  it "defaults to solrj_writer.batch_size more than 1" do
+    assert 1 < Traject::SolrJWriter.new("solr.url" => "http://example.org/solr").settings["solrj_writer.batch_size"].to_i
+  end
 
-      @writer = Traject::SolrJWriter.new(@settings)
+  describe "with no threading or batching" do
+    before do
+      @settings.merge!("solrj_writer.batch_size" => nil, "solrj_writer.thread_pool" => nil)
+      create_solrj_writer
     end
 
     it "writes a simple document" do
-      @writer.put "title_t" => ["MY TESTING TITLE"], "id" => ["TEST_TEST_TEST_0001"]
+      @writer.put context_with("title_t" => ["MY TESTING TITLE"], "id" => ["TEST_TEST_TEST_0001"])
       @writer.close
 
 
@@ -46,16 +101,29 @@ describe "Traject::SolrJWriter" do
         assert_kind_of org.apache.solr.client.solrj.impl.XMLResponseParser, @mock.parser
         assert_equal @settings["solr.url"], @mock.url
 
-        assert_equal 1, @mock.docs_added.length
-        assert_kind_of SolrInputDocument, @mock.docs_added.first
+        assert_equal 1, @mock.things_added.length
+        assert_kind_of SolrInputDocument, @mock.things_added.first
 
-        assert @mock.committed
         assert @mock.shutted_down
+      end
+    end
+
+    it "commits on close when so set" do
+      @settings.merge!("solrj_writer.commit_on_close" => "true")
+      create_solrj_writer
+
+      @writer.put context_with("title_t" => ["MY TESTING TITLE"], "id" => ["TEST_TEST_TEST_0001"])
+      @writer.close
 
-      else
+      # if it's not a mock, we don't really test anything, except that
+      # no exception was raised. oh well. If it's a mock, we can
+      # ask it.
+      if @mock
+        assert @mock.committed, "mock gets commit called on it"
       end
     end
 
+    test_handles_errors
 
 
     # I got to see what serialized marc binary does against a real solr server,
@@ -64,43 +132,78 @@ describe "Traject::SolrJWriter" do
     # real solr server set up.
     #
     # Not really a good test right now, just manually checking my solr server,
-    # using this to make the add reproducible at least. 
+    # using this to make the add reproducible at least.
     describe "Serialized MARC" do
       it "goes to real solr somehow" do
         record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
 
         serialized = record.to_marc # straight binary
-        @writer.put "marc_record_t" => [serialized], "id" => ["TEST_TEST_TEST_MARC_BINARY"]
+        @writer.put context_with("marc_record_t" => [serialized], "id" => ["TEST_TEST_TEST_MARC_BINARY"])
         @writer.close
       end
     end
-
   end
 
-end
+  describe "with batching but no threading" do
+    before do
+      @settings.merge!("solrj_writer.batch_size" => 5, "solrj_writer.thread_pool" => nil)
+      create_solrj_writer
+    end
 
-class MockSolrServer
-  attr_accessor :docs_added, :url, :committed, :parser, :shutted_down
+    it "sends all documents" do
+      docs = Array(1..17).collect do |i|
+        {"id" => ["item_#{i}"], "title" => ["To be #{i} again!"]}
+      end
 
-  def initialize(url)
-    @url =  url
-    @docs_added = []
-  end
+      docs.each do |doc|
+        @writer.put context_with(doc)
+      end
+      @writer.close
 
-  def add(solr_input_document)
-    docs_added << solr_input_document
-  end
+      if @mock
+        # 3 batches of 5, and the leftover 2 (16, 17)
+        assert_length 4, @mock.things_added
 
-  def commit
-    @committed = true
-  end
+        assert_length 5, @mock.things_added[0]
+        assert_length 5, @mock.things_added[1]
+        assert_length 5, @mock.things_added[2]
+        assert_length 2, @mock.things_added[3]
+      end
+    end
 
-  def setParser(parser)
-    @parser = parser
+    test_handles_errors
   end
 
-  def shutdown
-    @shutted_down = true
+  describe "with batching and threading" do
+    before do
+      @settings.merge!("solrj_writer.batch_size" => 5, "solrj_writer.thread_pool" => 2)
+      create_solrj_writer
+    end
+
+    it "sends all documents" do
+      docs = Array(1..17).collect do |i|
+        {"id" => ["item_#{i}"], "title" => ["To be #{i} again!"]}
+      end
+
+      docs.each do |doc|
+        @writer.put context_with(doc)
+      end
+      @writer.close
+
+      if @mock
+        # 3 batches of 5, and the leftover 2 (16, 17)
+        assert_length 4, @mock.things_added
+
+        # we can't be sure of the order under async,
+        # just three of 5 and one of 2
+        assert_length 3, @mock.things_added.find_all {|array| array.length == 5}
+        assert_length 1, @mock.things_added.find_all {|array| array.length == 2}
+      end
+    end
+
+    test_handles_errors
   end
 
-end
+end
+
+require 'thread' # Mutex

data/test/test_helper.rb

@@ -5,6 +5,16 @@ require 'minitest/spec'
 require 'traject'
 require 'marc'
 
+# keeps things from complaining about "yell-1.4.0/lib/yell/adapters/io.rb:66 warning: syswrite for buffered IO"
+# for reasons I don't entirely understand, involving yell using syswrite and tests sometimes
+# using $stderr.puts. https://github.com/TwP/logging/issues/31
+STDERR.sync = true
+
+# Hacky way to turn off Indexer logging by default, say only
+# log things higher than fatal, which is nothing. 
+require 'traject/indexer/settings'
+Traject::Indexer::Settings.defaults["log.level"] = "gt.fatal"
+
 def support_file_path(relative_path)
   return File.expand_path(File.join("test_support", relative_path), File.dirname(__FILE__))
 end
@@ -25,4 +35,44 @@ def assert_start_with(start_with, obj, msg = nil)
   msg ||= "expected #{obj} to start with #{start_with}"
 
   assert obj.start_with?(start_with), msg
+end
+
+# pretends to be a SolrJ HTTPServer-like thing, just kind of mocks it up
+# and records what happens and simulates errors in some cases. 
+class MockSolrServer
+  attr_accessor :things_added, :url, :committed, :parser, :shutted_down
+
+  def initialize(url)
+    @url =  url
+    @things_added = []
+    @add_mutex = Mutex.new
+  end
+
+  def add(thing)
+    @add_mutex.synchronize do # easy peasy threadsafety for our mock
+      if @url == "http://no.such.place"
+        raise org.apache.solr.client.solrj.SolrServerException.new("mock bad uri", java.io.IOException.new)
+      end
+
+      # simulate a multiple id error please
+      if [thing].flatten.find {|doc| doc.getField("id").getValueCount() != 1}
+        raise org.apache.solr.client.solrj.SolrServerException.new("mock non-1 size of 'id'")
+      else
+        things_added << thing
+      end
+    end
+  end
+
+  def commit
+    @committed = true
+  end
+
+  def setParser(parser)
+    @parser = parser
+  end
+
+  def shutdown
+    @shutted_down = true
+  end
+
 end

data/test/test_support/245_no_ab.marc

@@ -0,0 +1 @@
+01261cam  2200181   450000100070000000500170000700800410002403500110006503500080007604900200008410000600010424500120016430000190017652007500019560000600094591000250100599100490103014778120030102104200.0      m19171950xx                  eng d  a147781  aS#)  aJHWL [ARCHIVES]1 aLongcope, Warfield T.q(Warfield Theobald),d1877-1953.10kPapers.  a2.5 linear ft.  aLongcope was Director of the Department of Medicine at Johns Hopkins from 1922 to 1946. After graduating from Johns Hopkins, Longcope went to Pennsylvania Hospital in Philadelphia and ten years later to the College of Physicians and Surgeons at Columbia. During World War I, Longcope served in France as a colonel in the Medical Corps. At Hopkins Longcope took particular delight in the training of his resident staff. He had a reputation as a master diagnostician, and his own research included studies in Hodgkin's Disease, allergy and serum sickness, glomerulonephritis, sarcoidosis, and atypical pneumonia. The small collection of Longcope material contains correspondence from the World War I period, manuscripts on nephritis and offprints.10aLongcope, Warfield T.q(Warfield Theobald),d1877-1953.  a147781bHorizon bib#  falabwloccc. 1q0i273651lwarchivmwarchiv

data/test/test_support/880_with_no_6.utf8.marc

@@ -0,0 +1 @@
+02381cam a2200433   4500001000800000005001700008008004100025010001700066019002300083035001200106035001600118035005300134040004300187043001200230049000900242050002200251066000700273110005700280245019600337246006900533260004300602300002200645500016700667500003000834500004700864541006300911650003800974700004801012700002801060700001901088880008401107880029501191880005201486880027501538880004101813910002601854937005501880994001201935346856920121204134900.0811021m19639999ru       bc   000 0 rus    a   63058425   a2594268a231325908  a3468569  aocm07947282  a(OCoLC)7947282z(OCoLC)2594268z(OCoLC)231325908  aDLCbengcDLCdTJCdGZMdUPMdFUGdJHE  ae-ur---  aJHEE00aCD1734b.R87 1963  c(N1 6880-01aSoviet Union.bGlavnoe arkhivnoe upravlenie.106880-02aLichnye arkhivnye fondy v gosudarstvennykh khranilishchakh SSSR;bukazatelʹ.c[Redakt͡sionnai͡a kollegii͡a: I͡U.I. Gerasimova ... et al. Sostaviteli: Ė.V. Kolosova ... et al.].1 iTom 3 has title:aLichnye arkhivnye fondy v khranilishchakh SSSR  6880-03aMoska,c1962 [i.e. 1963]-<80 >  av. <1-3 >c27 cm.  6880-04aAt head of title: Glavnoe arkhivnoe upravlenie pri Sovete Ministrov SSSR. Gosudarstvennai͡a biblioteka SSSR imeni V.I. Lenina. Arkhiv Akademii nauk SSSR.  aTom 3 has imprint: Kniga.  aTom 3 edited by N.B. Volkova ... [et al.].  3Eisenhower copy:cGift ofaJeffrey Brooks;dFY2010.5MdBJ. 0aArchiveszSoviet UnionvCatalogs.1 aGerasimova, I͡U. I.q(I͡Ulii͡a Ivanovna)1 6880-05aKolosova, E. V.1 aVolkova, N. B.1 7110-01/(NaSoviet Union.bГлавное архивное управление.106245-02/(NaЛичные архивные фонды в государственных хранилищах СССР;bуказатель.c[Редакционная коллегия: Ю.И. Герасимова ... ет ал. Составители: Э.В. Колосова ... ет ал.].  6260-03/(NaМоска,c1962 [и.е. 1963]-<80>  6500-04/(NaAt head of title: Главное архивное управление при Совете Министров СССР. Государственная библиотека СССР имени В.И. Ленина. Архив Академии наук СССР.1 6700-05/(NaКолосова, Э. В.  a3468569bHorizon bib#  81985236aCD1734.R87 1963cc. 1qfalsemelscleoffs  aC0bJHE

data/test/test_support/bad_subfield_code.marc

@@ -0,0 +1 @@
+00582cam  2200217   4500001000800000005001700008008004100025035001200066035001400078035001900092035002300111040001300134049000800147050002000155100001800175245003300193260003900226300001000265910002600275991006300301117499920020328125800.0670101s1959    fr            000 0 fre    a1174999  aABK6934EI  a(MdBJ)99005528  a(CaOTULAS)00190819  aOTUbENG00aJHE 4aPQ2605 A873bC61 aCayrol, Jean.14aLes corps �etrangers, roman.  aParis��bEditions du Seuilc[1959]  a188p.  a1174999bHorizon bib#  aPH4605.C36 C6 1959flcbelccc. 1q0i2240984leoffsmelsc

data/test/test_support/date_resort_to_260.marc

@@ -0,0 +1 @@
+01739cas  2200457 a 4500001000800000005001700008008004100025010002200066022001400088035001200102035001400114035002000128035001800148035001400166037008300180040004300263042001300306043001200319049000900331050001900340074002300359082001900382086001500401222002300416245014400439246002700583246004000610260016700650300002500817310001100842500006000853500003200913650005600945710005301001710003801054710002101092710003201113780006201145910002601207937004801233100206120050505054200.0101719c19uu9999dcuar1    s  f0   a0eng d  a   80643740 //r810 a0270-1499  a1002061  aAEV0113GD  a(OCoLC)06213876  a(GPO)82028743  a(MdBJXXX)  bU.S. Dept. of Justice, Drug Enforcement Administration, Washington, D.C. 20537  aGPOcGPOdDLCdNSDdDLCdGPOdOCLdGPO  alcansdp  an-us---  aJHE1  aHV5825b.U565c  a968-G (microfiche)  a352.2/932/09730 aJ 24.19/2: 0aDAWN annual report00aDAWN annual report /csubmitted by IMS America, Ltd. ; submitted to Drug Enforcement Administration [and] National Institute on Drug Abuse.10aD.A.W.N. annual report14aProject DAWN annual reportf<1978->  aWashington, D.C. :bU.S. Dept. of Justice, Drug Enforcement Administration,bU.S. Dept. of Health, Education and Welfare, National Institute on Drug Abuse,c1980-  av. ;bill. ;c28 cm.  aAnnual  aAt head of title <, 1978->: Drug Abuse Warning Network.  aDescription based on: 1979. 0aDrug abusezUnited StatesvStatisticsvPeriodicals.1 aUnited States.bDrug Enforcement Administration.2 aNational Institute on Drug Abuse.2 aIMS America Ltd.2 aDrug Abuse Warning Network.00tPhase report - Drug Abuse Warning Networkw(OCoLC)4424916  a1002061bHorizon bib#  8898944aJ 24.19/2:cc. 1qfalsemegovlegpo

data/test/test_support/date_type_r_missing_date2.marc

@@ -0,0 +1 @@
+00974nam  22002771  4500001000800000005001700008008004100025010001700066035001200083035001400095035001900109040000900128049000800137050002200145100003300167245009000200260006400290300003800354490003600392600007600428610003000504650003000534830004500564910002600609991006100635100104119971120112300.0821029r1957    flu           000 0 eng    a   57009033   a1001041  aABD1688EI  a(MdBJ)57009033  cCarP00aJHE0 aBR757b.P37 1608a1 aParsons, Robert,d1546-1610.14aThe judgment of a Catholicke English-man living in banishment for his religion (1608)  aGainesville, Fla.,bScholars' Facsimiles & Reprints,c1957.  axvii p., facsim. : 128 p.c23 cm.1 aScholars' facsimiles & reprints00aJamesbI,cKing of England,d1566-1625.tTriplici nodo, triplex cuneus.20aCatholic ChurchzEngland. 0aOath of allegiance, 1606. 0aScholars' Facsimiles & Reprints (Series)  a1001041bHorizon bib#  aBR757 .P37 1957flcbelccc. 1q0i2046757lemainmemsel

data/test/test_support/date_with_u.marc

@@ -0,0 +1 @@
+01014nas a2200253 a 4500001000800000005001700008008004100025035001200066035001400078035002700092040001500119049000800134130005500142245003000197260010100227300001700328500006500345541011400410650002800524710007000552710006000622910002600682937005200708100077619971120111500.0880525d184u18uuenkmr p       0   a0eng d  a1000776  aADK9570EI  a(CStRLIN)MDJGADK9570-S  aMdBJcMdBJ00aJHE0 aEducational circular (Cheltenham, Gloucestershire)14aThe educational circular.  aCheltenham :bPrinted for the Committee for Promoting Universal Sympathy and Charity by Leonard,  av. ;c22 cm.  aDescription based on: No. 2 (Feb. 1841); title from caption.  3 Eisenhower copy:c Purchased with funds from thea Abram G. and David Hutzler Economic Classics Fund.5 MdBJ 0aEducationvPeriodicals.2 aAbram G. and David Hutzler Collection of Economic Classics.5MdBJ2 aCommittee for Promoting Universal Sympathy and Charity.  a1000776bHorizon bib#  8146026aL16.E4falacc. 1qfalsememselleshtlr

data/test/test_support/demo_config.rb

@@ -0,0 +1,153 @@
+#require 'traject/macros/marc21'
+#require 'traject/macros/basic'
+#extend Traject::Macros::Marc21
+#extend Traject::Macros::Basic
+
+require 'traject/macros/marc21_semantics'
+extend  Traject::Macros::Marc21Semantics
+
+require 'traject/macros/marc_format_classifier'
+extend Traject::Macros::MarcFormats
+
+settings do
+  #provide "solr.url", "http://catsolrmaster.library.jhu.edu:8985/solr/master_prod"
+
+  provide "solr.url", "http://blacklight.mse.jhu.edu:8983/solr/prod"
+  provide "solrj_writer.parser_class_name", "XMLResponseParser"
+
+  provide "solrj_writer.commit_on_close", true
+
+  #require 'traject/marc4j_reader'
+  #store "reader_class_name", "Marc4JReader"
+end
+
+to_field "id", extract_marc("001", :first => true) do |marc_record, accumulator, context|
+  accumulator.collect! {|s| "bib_#{s}"}
+
+  # A way to intentionally add errors
+  #if context.position % 10 == 0
+    # intentionally add another one to error
+  #  accumulator << "ANOTHER"
+  #end
+
+end
+
+to_field "source",              literal("traject_test_last")
+
+to_field "marc_display",        serialized_marc(:format => "binary", :binary_escape => false)
+
+to_field "text",                extract_all_marc_values
+
+to_field "text_extra_boost_t",  extract_marc("505art")
+
+to_field "publisher_t",         extract_marc("260abef:261abef:262ab:264ab")
+
+to_field "language_facet",      marc_languages
+
+to_field "format",              marc_formats
+
+
+to_field "isbn_t",              extract_marc("020a:773z:776z:534z:556z")
+to_field "lccn",                extract_marc("010a")
+
+to_field "material_type_display", extract_marc("300a", :seperator => nil, :trim_punctuation => true)
+
+to_field "title_t",             extract_marc("245ak")
+to_field "title1_t",            extract_marc("245abk")
+to_field "title2_t",            extract_marc("245nps:130:240abcdefgklmnopqrs:210ab:222ab:242abcehnp:243abcdefgklmnopqrs:246abcdefgnp:247abcdefgnp")
+to_field "title3_t",            extract_marc("700gklmnoprst:710fgklmnopqrst:711fgklnpst:730abdefgklmnopqrst:740anp:505t:780abcrst:785abcrst:773abrst")
+to_field "title3_t" do |record, accumulator|
+  # also add in 505$t only if the 505 has an $r -- we consider this likely to be
+  # a titleish string, if there's a 505$r
+  record.each_by_tag('505') do |field|
+    if field['r']
+      accumulator.concat field.subfields.collect {|sf| sf.value if sf.code == 't'}.compact
+    end
+  end
+end
+
+to_field "title_display",       extract_marc("245abk", :trim_puncutation => true, :first => true)
+to_field "title_sort",          marc_sortable_title
+
+to_field "title_series_t",      extract_marc("440a:490a:800abcdt:400abcd:810abcdt:410abcd:811acdeft:411acdef:830adfgklmnoprst:760ast:762ast")
+to_field "series_facet",        marc_series_facet
+
+to_field "author_unstem",       extract_marc("100abcdgqu:110abcdgnu:111acdegjnqu")
+
+to_field "author2_unstem",      extract_marc("700abcdegqu:710abcdegnu:711acdegjnqu:720a:505r:245c:191abcdegqu")
+to_field "author_display",      extract_marc("100abcdq:110:111")
+to_field "author_sort",         marc_sortable_author
+
+
+to_field "author_facet",        extract_marc("100abcdq:110abcdgnu:111acdenqu:700abcdq:710abcdgnu:711acdenqu", :trim_punctuation => true)
+
+to_field "subject_t",           extract_marc("600:610:611:630:650:651avxyz:653aa:654abcvyz:655abcvxyz:690abcdxyz:691abxyz:692abxyz:693abxyz:656akvxyz:657avxyz:652axyz:658abcd")
+
+to_field "subject_topic_facet", extract_marc("600abcdtq:610abt:610x:611abt:611x:630aa:630x:648a:648x:650aa:650x:651a:651x:691a:691x:653aa:654ab:656aa:690a:690x",
+          :trim_puncutation => true, ) do |record, accumulator|
+  #upcase first letter if needed, in MeSH sometimes inconsistently downcased
+  accumulator.collect! do |value|
+    value.gsub(/\A[a-z]/) do |m|
+      m.upcase
+    end
+  end
+end
+
+to_field "subject_geo_facet",   marc_geo_facet
+to_field "subject_era_facet",   marc_era_facet
+
+# not doing this at present. 
+#to_field "subject_facet",     extract_marc("600:610:611:630:650:651:655:690")
+
+to_field "published_display", extract_marc("260a", :trim_punctuation => true)
+
+to_field "pub_date",          marc_publication_date
+
+# LCC to broad class, start with built-in from marc record, but then do our own for local
+# call numbers.
+lcc_map = Traject::TranslationMap.new("lcc_top_level")
+to_field "discipline_facet",  marc_lcc_to_broad_category(:default => nil) do |record, accumulator|
+  # add in our local call numbers
+  accumulator.concat(
+    Traject::MarcExtractor.new(record, "991:937").collect_matching_lines do |field, spec, extractor|
+        # we output call type 'processor' in subfield 'f' of our holdings
+        # fields, that sort of maybe tells us if it's an LCC field.
+        # When the data is right, which it often isn't.
+      call_type = field['f']
+      if call_type == "sudoc"
+        # we choose to call it:
+        "Government Publication"
+      elsif call_type.nil? || call_type == "lc" || field['a'] =~ Traject::Macros::Marc21Semantics::LCC_REGEX
+        # run it through the map
+        s = field['a']
+        s = s.slice(0, 1) if s
+        lcc_map[s]
+      else
+        nil
+      end
+    end.compact
+  )
+
+  # If it's got an 086, we'll put it in "Government Publication", to be
+  # consistent with when we do that from a local SuDoc call #.
+  if Traject::MarcExtractor.extract_by_spec(record, "086a", :seperator =>nil).length > 0
+    accumulator << "Government Publication"
+  end
+
+  accumulator.uniq!
+
+  if accumulator.empty?
+    accumulator << "Unknown"
+  end
+end
+
+to_field "instrumentation_facet",       marc_instrumentation_humanized
+to_field "instrumentation_code_unstem", marc_instrument_codes_normalized
+
+to_field "issn",                extract_marc("022a:022l:022y:773x:774x:776x", :seperator => nil)
+to_field "issn_related",        extract_marc("490x:440x:800x:400x:410x:411x:810x:811x:830x:700x:710x:711x:730x:780x:785x:777x:543x:760x:762x:765x:767x:770x:772x:775x:786x:787x", :seperator => nil)
+
+to_field "oclcnum_t",           oclcnum
+
+to_field "other_number_unstem", extract_marc("024a:028a")
+

data/test/test_support/emptyish_record.marc

@@ -0,0 +1 @@
+00427nas a22001695? 4500001000800000005001700008008004100025035001200066035001400078035001200092049000800104245004000112260003500152910002600187937003800213997000600251100016519971120111300.0880524c|||||||||||u| m||||||||   ||||||d  a1000165  aADK9358EI  aADK935800aJHE00aCollection la philosophie en effet.  aParis : Editions Galilee, 19 -  a1000165bHorizon bib#  8897144cc. 1qfalsememsellemain  bp

data/test/test_support/louis_armstrong.marc

@@ -0,0 +1 @@
+02551cjm a2200649 a 4500001000800000005001700008007001500025008004100040010002300081019001300104028002900117033002300146035001200169035001200181035001600193040001300209048001300222049000900235050001000244050002800254100003900282245004000321260004600361300005700407490001400464500014300478505050200621511005501123518002401178650002001202650002101222650002101243740003601264740003001300740002701330740002401357740001801381740003401399740001601433740002201449740002001471740002101491740003601512740002901548740002001577740001501597740003601612740002101648740001401669740005001683740001401733740001701747740003701764830001501801910002601816991005901842204330820080415184300.0sdubmmennmplu-861105p19851935iluppn   fi         eng    a   86754802 /R/r98  a1311821902aSTBB-22bTime-Life Music2 a1935----a1944----  a2043308  a2043308  aocm15164034  aDLCcDLC  bbb01aoe  aJHPA10aM135600aTime-Life Music STBB-221 aArmstrong, Louis,d1901-1971.4prf10aLouis Armstrongh[sound recording].  aChicago, Ill. :bTime-Life Music,cp1985.  a2 sound discs :banalog, 33 1/3 rpm, mono. ;c12 in.1 aBig bands  aBiographical and program notes by Stanley Dance ([2] p. : 1 port.) in container; personnel and original issue and matrix no. on container.0 aThe music goes 'round and 'round -- I'm in the mood for love -- You are my lucky star -- Something tells me -- Ol' man Mose -- Struttin' with some barbecue -- Sunshowers -- Jeepers creepers -- So little time -- True confession -- When the saints go marching in -- On the sentimental side -- Love walked in -- Confessin' -- I've got a pocketful of dreams -- Wolverine blues -- Solitude -- (I'll be glad when you're dead) you rascal you -- Coquette -- Whatcha say -- On the sunny side of the street.0 aLouis Armstrong, trumpet and vocals, and his band.  aRecorded 1935-1944. 0aBig band music. 0aJazzy1931-1940. 0aJazzy1941-1950.4 aThe music goes round and round.0 aI'm in the mood fro love.0 aYou are my lucky star.0 aSomething tells me.0 aOl' man Mose.0 aStruttin' with some barbecue.0 aSunshowers.0 aJeepers creepers.0 aSo little time.0 aTrue confession.0 aWhen the saints go marching in.0 aOn the sentimental side.0 aLove walked in.0 aConfessin'0 aI've got a pocketful of dreams.0 aWolverine blues.0 aSolidute.0 aI'll be glad when you're dead you rascal you.0 aCoquette.0 aWhatcha say.0 aOn the sunny side of the street. 0aBig bands.  a2043308bHorizon bib#  aA1.350 .B499 A7fnonebavcc. 1q0i3246401lasalmafl

data/test/test_support/manuscript_online_thesis.marc

@@ -0,0 +1 @@
+01433ntm a2200337Ka 4500001000800000005001700008006001900025007001400044007001500058008004100073035001700114040001300131049000900144100002800153245013900181260001000320300003700330500001000367500002400377502005300401504005400454533010600508590005900614690007200673856008900745910002600834991006500860991009100925991006701016994001201083259448320060313115742.0m        d        hd afb---bacacr un         060313s2005    xx a    abm   000 0 eng d  aocm64633779   aJHEcJHE  aJHEE1 aSchowalter, Kathleen S.10aCapetian women and their booksh[microform] :bart, ideology, and dynastic continuity in medieval France /cby Kathleen S. Schowalter.  c2005.  axv, 364 leaves :bill. ;c28 cm.  aVita.  aU.M.I. no. 3172690.  aThesis (Ph. D.)--Johns Hopkins University, 2005.  aIncludes bibliographical references (l. 236-277).  aMicrofilm.bAnn Arbor, Mich. :cUniversity Microfilm International,d2005.e1 microfilm reel ; 35 mm.  aEISENHOWER: Held in print, microfilm, and Web version.  aJohns Hopkins UniversityxDissertations and essays (History of Art)41uhttp://wwwlib.umi.com/dissertations/dlnow/3172690zAvailable to US Hopkins community  a2594483bHorizon bib#  aFILM no. D 4257falabeformatcc. 1q0i4286162leofavmelsc  aJHU THESIS Schowalter, Kathleen S 2005falabeothercc. 1q0i4286158leofdissmelscsp  aWorld Wide Webfalabeformatcc. 1q0i4286174ledispromemsel  aC0bJHE

data/test/test_support/microform_online_conference.marc

@@ -0,0 +1 @@
+02093cam a2200481 a 450000100070000000500170000700600190002400700070004300700140005000800410006401000170010502000280012202000220015002000220017203500110019403500140020503500270021904000190024604900080026505000200027308200140029311100910030724502880039826000680068630000350075450000410078950000380083050000780086850400410094653300560098753800590104359000480110265000410115065000330119165000260122465000270125071000270127771000540130485601090135891000250146793700570149293700620154944661420051004210200.0m        d        cr unuhe|bmu---baca900705s1988    dcua    bb    101 0 eng    a   87083478   a0818648376 (microfiche)  a0818608374 (pbk.)  a0818688378 (hard)  a446614  aADY2595EI  a(CStRLIN)MDJGADY2595-B  dNNdMdBJdMdBJ00aJHE0 aQ334b.C66 19880 a006.32192 aConference on Artificial Intelligence Applicationsn(4th :d1988 :cSan Diego, Calif.)10aProceedings, the fourth Conference on Artificial Intelligence Applicationsh[microform] :bSheraton Harbor Island Hotel, San Diego, California, March 14-18, 1988 /csponsored by the Computer Society of the IEEE in cooperation with the American Association of Artificial Intelligence.  aWashington, D.C. :bComputer Society Press of the IEEE,cc1988.  axvii, 440 p. :bill. ;c28 cm.  a"Computer Society order number 837."  a"IEEE catalog number 88CH2552-8."  a"Sheraton Harbor Island Hotel, San Diego, California, March 14-18, 1988."  aIncludes bibliographies and indexes.  aMicrofiche.e5 microfiches : negative ; 11 x 15 cm.  aMode of access for electronic version: World Wide Web.  aEISENHOWER: Held in print and Web versions. 0aArtificial intelligencevCongresses. 0aComputer visionvCongresses. 0aRoboticsvCongresses. 0aReasoningvCongresses.2 aIEEE Computer Society.2 aAmerican Association for Artificial Intelligence.41zOnline version:uhttp://ieeexplore.ieee.org/servlet/opac?punumber=195zAvailable to US Hopkins community  a446614bHorizon bib#  81909587aWorld Wide Webcc. 1qfalsememselleieeexp  8357414aMicro- fiche C no. 1695cc. 1qfalsemelscleofav

data/test/test_support/multi_era.marc

@@ -0,0 +1 @@
+01358mam a22003498a 4500001000800000005001700008008004100025010001700066020001500083035001200098035001400110035002000124040001900144043001200163049000900175050002000184082001800204100002100222245008700243260007700330263000900407300002600416504006600442650007200508650006700580650005400647651009000701651007400791651005500865910002600920991006200946141037219971122050700.0920406s1992    enk      b    001 0 eng    a   92013022   a0198128835  a1410372  aAFJ4571EI  a(OCoLC)25707456  aDLCcDLCdMdBJ  ae-uk---  aJHEE00aPR435bC67 199200a820.9/3582201 aCorns, Thomas N.10aUncloistered virtue :bEnglish political literature, 1640-1660 /cThomas N. Corns.  aOxford :bClarendon Press ;aNew York :bOxford University Press,c1992.  a9207  axii, 333 p. ;c23 cm.  aIncludes bibliographical references (p. [313]-327) and index. 0aEnglish literatureyEarly modern, 1500-1700xHistory and criticism. 0aPolitics and literaturezGreat BritainxHistoryy17th century. 0aPolitical poetry, EnglishxHistory and criticism. 0aGreat BritainxHistoryyPuritan Revolution, 1642-1660xLiterature and the Revolution. 0aGreat BritainxHistoryyCivil War, 1642-1649xLiterature and the war. 0aGreat BritainxPolitics and governmenty1642-1660.  a1410372bHorizon bib#  aPO 435 .C67 1992flcbelccc. 1q0i2499314lemainmemsel

data/test/test_support/multi_geo.marc

@@ -0,0 +1 @@
+01005cam a2200289 i 450000100070000000500170000700800410002403500110006503500140007603500250009004300300011504900080014505000330015308200230018610000230020924500930023226000780032530000570040349000310046050000220049150400300051365000240054365100270056783000270059491000250062199100690064674641419971119015300.0750908s1961    njuaf    b    001 0 eng    a746414  aAAT2611EI  a(MdBJ)75322654 //r82  ae------aaw-----aff-----00aJHE0 aDF287.A23bA5 vol. 7aNK4675  a938/.5 sa738.3/831 aPerlzweig, Judith.10aLamps of the Roman period, first to seventh century after Christ /cby Judith Perlzweig.  aPrinceton, N.J. :bAmerican School of Classical Studies at Athens,c1961.  axiv, 240 p., [27] leaves of plates :bill. ;c31 cm.1 aThe Athenian Agora ;vv. 7  aIncludes indexes.  aBibliography: p. [x]-xiv. 0aLampszItalyzRome. 0aAgora (Athens, Greece) 0aAthenian Agora;vv. 7.  a746414bHorizon bib#  aPC5376.A7 A5 1953 Q v.7flcbelccc. 1q0i1753457lemainmemsel

data/test/test_support/musical_cage.marc

@@ -0,0 +1 @@
+03531cjm a2200721Ia 4500001000800000005001700008007001500025008004100040024001700081028001800098035001600116040003100132045003800163048000700201048000900208048001500217048001300232048001500245048000900260049000900269100001500278240002300293245006200316260003600378300006200414306003500476440002900511500005200540500037200592500004900964505029301013511043101306518024801737590004801985650002702033650001602060650001902076650001002095650003302105650003702138700002502175700002002200700001802220700002302238700002702261700001702288700002002305700001502325700001702340700002802357700002402385700002202409700003002431700004102461700005702502700004102559700004102600710004402641710003002685910002602715991005602741994001202797235068120031002121540.0sd fsngnnmmned010111s2000    caumun   eh         eng d1 a02147508806502a8806bCambria  aocm45703484  aINDcINDdOCLCQdIXAdJHPA1 bd1961bd1973bd1957bd1970bd1954  aon  aka01  ava01avd01  asa01aoy  avd01atb01  avn12  aJHPA1 aCage, John10aAtlas eclipticalis10aLou Harrison, Harry Partch, John Cageh[sound recording].  aLomita, Cal. :bCambria,c2000.  a2 sound discs (121 min.) :bdigital, stereo. ;c4 3/4 in.  a004548a002818a000950a003342 0aComposer portrait series  aTitle from disc A; disc B has title: John Cage.  a1st work performed by flute, clarinet, alto saxophone, bassoon/contra-bassoon, 2 horns, trombone, 2 violins, viola, double bass, guitar, percussion (2), and conductor (Winter music by piano, Songbooks by soprano & baritone); 2nd work for violin and American gamelan; 3rd work for baritone and adapted guitar; the last work for 12 male speaker/vocalists and conductor.  aProgram notes (12 p. : ports.) in container.0 aAtlas elipticals (with Winter music and Songbooks) / John Cage (48:58) -- Suite for violin and American gamelan / Lou Harrison (28:18) -- Barstow : 8 hitchhiker inscriptions from a highway railing at Barstow, California / Harry Partch (9:50) -- Lecture on the weather / John Cage (33:42).0 aMembers of the Southwest Chamber Music: Phyllis Bryn-Julson, soprano ; Michael Ingham, baritone ; Jim Foschia, clarinet ; Leslie Lashinsky, bassoons ; Jeff von der Schmidt, horn ; Don Ambronson, Amy Sims, violins ; Jan Karlin, viola ; Tom Peters, double bass ; Gayle Blankenburg, piano ; Stephen L. Mosko, conductor ; mentorship students from John Muir High School, Pasadena ; Jim Foschia, director ; CalArts Gamelan Ensemble.  aAtlas elipticalis (with Winter Music, Songbooks) recorded on 13 Apr. 1999; Suite and Barstow on 23 Feb, 1999; Lecture on weather on 16 March 1999, all at the Herbert Zipper Concert Hall, Colburn School of the Performing Arts, Los Angeles, Cal.  aFRIEDHEIM: The gift of Phyllis Bryn-Julson. 0aInstrumental ensembles 0aPiano music 0aAleatory music 0aSongs 0aSuites (Violin with gamelan) 0aSongs (Medium voice) with guitar1 aBryn-Julson, Phyllis1 aIngham, Michael1 aFoschia, Jim.1 aLashinsky, Leslie.1 aVon der Schmidt, Jeff.1 aFox, Stuart.1 aAmbronson, Don.1 aSims, Amy.1 aKarlin, Jan.1 aPeters, Tom,cmusician.1 aBlankenburg, Gayle.1 aMosko, Stephen L.12aCage, John.tWinter music12aCage, John.tSongbooks.kSelections.12aHarrison, Lou,d1917-2003.tSuites,mviolin, gamelan12aPartch, Harry,d1901-1974.tBarstow.12aCage, John.tLecture on the weather.2 aSouthwest Chamber Music (Musical group)2 aCalArts Gamelan Ensemble.  a2350681bHorizon bib#  aCD 312falabaloccc. 1, v. 6q0i3894474lacdmafl  aE0bJHP

data/test/test_support/one-marc8.mrc

@@ -0,0 +1 @@
+00663cam  2200217Ia 4500001000800000005001700008008004100025020001500066035001600081040001300097049000900110090002200119100002000141245010100161260004500262300002100307650001800328910002600346991006100372994001200433219638420010919155148.0010626s2000    bl            000 0 por d  a8501058785  aocm47198253  aIXAcIXA  aJHEE  aJZ1308b.S26 20001 aSantos, M�ilton10aPor uma outra globaliza�c�ao :bdo pensamemto �unico �a consci�encia universal /cMilton Santos.  aRio de Janeiro :bEditora Record,c2000.  a174 p. ;c21 cm. 0aGlobalization  a2196384bHorizon bib#  aJZ1308.S26 2000flcbelccc. 1q0i3557489lemainmemsel  aX0bJHE

data/test/test_support/online_only.marc

@@ -0,0 +1 @@
+01173nam a22003012u 4500001000800000005001700008006001900025007001500044008004100059020003000100035001400130035002100144040003500165050002300200082001700223100001800240245005700258260003600315300003100351500005200382650003000434650003200464655002200496776017900518856008200697910002600779991006600805426986720120912153319.0m        d        cr -n---------120912s2011|||||||     s|||||||||||eng|d  a9780465026340c16.99 (NL)  aEBL894955  a(OCoLC)798534244  aAU-PeELbengcAU-PeELdAU-PeEL 4aZ7164.O7 B323 201100a016.658a6581 aBooks, Basic.14aThe Best Business Books Everh[electronic resource].  aNew York :bBasic Books,c2011.  a1 online resource (281 p.)  aDescription based upon print version of record. 4aBusiness -- Bibliography. 4aManagement -- Bibliography. 0aElectronic books.08iPrint version:aBooks, BasictThe Best Business Books Ever : The Most Influential Management Books You'LL Never Have Time to ReaddNew York : Basic Books,c2011z978046502236640zClick here to view bookuhttp://JHU.eblib.com/patron/FullRecord.aspx?p=894955  a4269867bHorizon bib#  aWorld WideWebfalabeformatcc. 1q0i6290471leeblstlmemsel

data/test/test_support/packed_041a_lang.marc

@@ -0,0 +1 @@
+00891cam a2200253   450000100070000000500170000700800410002401500120006502000150007703500110009203500140010303500190011704100230013604900080015905000160016708200100018310000190019324502390021226000530045130000280050465000140053291000250054699100660057158749319971118093200.0711130s1971    ne            000 0 engx   aNe71-38  a0444409041  a587493  aAAL0723EI  a(MdBJ)751354830 aengfregeritasparus00aJHE0 aPN6404b.G6  a398.91 aGluski, Jerzy.10aProverbs. Proverbes. Sprichwörter. Proverbi. Proverbios. Poslovit͡sy (romanized form)bA comparative book of English, French, German, Italian, Spanish and Russian proverbs with a Latin appendix.cCompiled and edited by Jerzy Gluski.  aAmsterdam,aNew York,bElsevier Pub. Co.,c1971.  axxxviii, 448 p.c20 cm. 0aProverbs.  a587493bHorizon bib#  aPN 6404 .G57 1971flcbelc1cc. 2q0i1556248legerfblmemsel

data/test/test_support/the_business_ren.marc

@@ -0,0 +1 @@
+01635cas a2200409 a 4500001000800000005001700008006001900025007001500044008004100059010001700100022002500117035001500142035002300157035002000180037011900200040004800319042001400367043001200381050001700393082001500410222003900425245006300464260006000527310001400587362003300601538003600634650005400670650005500724650006700779710003600846710002100882710002900903850000800932856011500940856014401055910002601199354932120111006075220.0m        d        cr  n         060315c20069999cauqr pss     0   a0eng    a  2006214489  l1930-7462y1930-7462  assj0054406  a(WaSeSS)ssj0054406  a(OCoLC)64686176  bThe Business Renaissance Institute/King International Group, 527 South Lake Ave., Ste. 102, Pasadena, CA 9110-3529  aNSDPcNSDPdDLCdOCoLCdCU-SdOCoLCdWaSeSS  ansdpapcc  an-us---00aHD6951b.B8710a306.36213 4aThe Business renaissance quarterly04aThe Business renaissance quarterlyh[electronic resource].  aPasadena, CA :bBusiness Renaissance Institute,cc2006-  aQuarterly0 aVol. 1, no. 1 (summer 2006)-  aMode of access: World Wide Web. 0aQuality of work lifezUnited StatesvPeriodicals. 0aIndustrial managementzUnited StatesvPeriodicals. 0aSocial responsibility of businesszUnited StatesvPeriodicals.2 aBusiness Renaissance Institute.2 aProQuest Central2 aBusiness Source Complete  aDLC403Full text available from ProQuest Central: 04/01/2006 to presentuhttp://search.proquest.com/publication/39705403Full text available from Business Source Complete: 06/01/2006 to presentuhttp://search.ebscohost.com/direct.asp?db=bth&jid=2Q82&scope=site  a3549321bHorizon bib#

data/test/translation_map_test.rb

@@ -67,6 +67,14 @@ describe "TranslationMap" do
     assert_equal "value1", map["key1"]
   end
 
+  it "finds .properties defn" do 
+    map =Traject::TranslationMap.new("properties_map")
+
+    assert_equal "Value1", map["key1"]
+    assert_equal "Value2", map["key2"]
+    assert_equal "Multi word value", map["key3"]   
+  end
+
   it "can use a hash instance too" do
     map = Traject::TranslationMap.new(
       "input_value" => "output_value"

data/test/translation_maps/properties_map.properties

@@ -0,0 +1,5 @@
+# This is a comment
+
+ key1 = Value1
+        key2:Value2  
+ key3                  :Multi word value   

data/traject.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "marc", ">= 0.5.1"
   spec.add_dependency "hashie", ">= 2.0.5", "< 2.1" # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0"   # command line parsing
-  
+  spec.add_dependency "yell" # logging
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/vendor/marc4j/README.md

@@ -0,0 +1,17 @@
+The marc4j.jar file that is in `./lib` was created by:
+
+Checking out the marc4j source from github.com/marc4j/marc4j, and building
+with `ant` on 23 July 2013.
+
+That produced the jar you see, with the name it has. (I am not sure the version
+in the jar name is accurate, it's not actually the beta1 release, but the RC1
+release, or maybe even a subsequent release?)
+
+It can be regenerated by doing the same.
+
+Bundling the marc4j jar with our traject gem is not neccesarily the
+best way to go, but it's what we're doing now. See top-level README
+TODO.
+
+(You can use your own custom marc4j.jar by using a runtime setting,
+see Marc4JReader class docs. )