traject 0.16.0 → 0.17.0

data/lib/traject/translation_map.rb

@@ -13,19 +13,28 @@ module Traject
   # or array of strings.
   #
   # What makes it more useful than a stunted hash is it's ability to load
-  # the hash definitions from configuration files, either pure ruby, 
-  # yaml, or java .properties.  (Limited basic .properties, don't try any fancy escaping please,
-  # no = or : in key names, no split lines.)
+  # the hash definitions from configuration files, either pure ruby,
+  # yaml, or (limited subset of) java .properties file.
   #
-  #     TranslationMap.new("dir/some_file")
+  # traject's `extract_marc` macro allows you to specify a :translation_map=>filename argument
+  # that will automatically find and use a translation map on the resulting data:
   #
-  # Will look for a file named `some_file.rb` or `some_file.yaml` or `some_file.properties`, 
-  # somewhere in the ruby $LOAD_PATH in a `/translation_maps` subdir. 
-  # * Looks for "/translation_maps" subdir in load paths, so
+  #     extract_marc("040a", :translation_map => "languages")
+  #
+  # Or you can always create one yourself and use it how you like:
+  #
+  #     map = TranslationMap.new("languages")
+  #
+  # In either case, TranslationMap will look for a file named, in that example,
+  # `languages.rb` or `languages.yaml` or `languages.properties`,
+  # somewhere in the ruby $LOAD_PATH in a `/translation_maps` subdir.
+  #
+  # * Also looks for "/translation_maps" subdir in load paths, so
   #   for instance you can have a gem that keeps translation maps
-  #   in ./lib/translation_maps, and it Just Works. 
+  #   in ./lib/translation_maps, and it Just Works.
+  #
   # * Note you do NOT supply the .rb, .yaml, or .properties suffix yourself,
-  # it'll use whichever it finds (allows calling code to not care which is used).
+  #  it'll use whichever it finds (allows calling code to not care which is used).
   #
   # Ruby files just need to have their last line eval to a hash. They file
   # will be run through `eval`, don't do it with untrusted content (naturally)
@@ -33,7 +42,7 @@ module Traject
   # You can also pass in a Hash for consistency to TranslationMap.new, although
   # I don't know why you'd want to.
   #
-  # == Special default handling
+  # ## Special default handling
   #
   # The key "__default__" in the hash is treated specially. If set to a string,
   # that string will be returned by the TranslationMap for any input not otherwise
@@ -50,19 +59,20 @@ module Traject
   #    TranslationMap.new("something", :default => "foo")
   #    TranslationMap.new("something", :default => :passthrough)
   #
-  # == Output: String or array of strings
+  # ## Output: String or array of strings
   #
   # The output can be a string or an array of strings, or nil.  It should not be anything else.
   # When used with the #translate_array! method, one string can be replaced by multiple values
   # (array of strings) or removed (nil)
   #
-  # There's no way to specify multiple return values in a .properties, use .yaml or .rb for that. 
+  # There's no way to specify multiple return values in a .properties, use .yaml or .rb for that.
+  #
+  # ## Caching
   #
-  # == Caching
   # Lookup and loading of configuration files will be cached, for efficiency.
   # You can reset with `TranslationMap.reset_cache!`
   #
-  # == YAML example:
+  # ## YAML example:
   #
   #     key: value
   #     key2: value2 multiple words fine
@@ -71,13 +81,35 @@ module Traject
   #       - array
   #       - of
   #       - values look like this
+  #
+  # ## Alternatives
+  # `Traject::TranslationMap` provides an easy way to deal with the most common translation case:
+  # simple key-value stores with optional default values.
+  #
+  # If you need more complex translation, you can simply use `#map!`
+  # or its kin to work on the `accumulator` in a block
+  #
+  #
+  #
+  #     # get a lousy language detection of any vernacular title
+  #     require 'whatlanguage'
+  #     wl = WhatLanguage.new(:all)
+  #     to_field 'vernacular_langauge', extract_marc('245', :alternate_script=>:only) do |rec, acc|
+  #       # accumulator is already filled with the values of any 880s that reference a 245 because
+  #       # of the call to #extract_marc
+  #       acc.map! {|x| wl.language(x) }
+  #       acc.uniq!
+  #     end
+  # Within the block, you may also be interested in using:
+  # * a case-insentive hash, perhaps like [this one](https://github.com/junegunn/insensitive_hash)
+  # * a [MatchMap](https://github.com/billdueber/match_map), which implements pattern-matching logic similar to solrmarc's pattern files
   class TranslationMap
     class Cache
       def initialize
-        @cached = Hash.new 
+        @cached = Hash.new
       end
 
-      # Returns an actual Hash -- or nil if none found. 
+      # Returns an actual Hash -- or nil if none found.
       def lookup(path)
         unless @cached.has_key?(path)
           @cached[path] = _lookup!(path)
@@ -86,9 +118,9 @@ module Traject
       end
 
       # force lookup, without using cache.
-      # used by cache. Returns the actual hash. 
-      # Returns nil if none found. 
-      # May raise on syntax error in file being loaded. 
+      # used by cache. Returns the actual hash.
+      # Returns nil if none found.
+      # May raise on syntax error in file being loaded.
       def _lookup!(path)
         found = nil
 
@@ -110,7 +142,7 @@ module Traject
         end
 
         # Cached hash can't be mutated without weird consequences, let's
-        # freeze it! 
+        # freeze it!
         found.freeze if found
 
         return found
@@ -163,20 +195,20 @@ module Traject
     alias_method :map, :[]
 
     # Returns a dup of internal hash, dup so you can modify it
-    # if you like. 
+    # if you like.
     def to_hash
       @hash.dup
     end
 
     # Run every element of an array through this translation map,
     # return the resulting array. If translation map returns nil,
-    # original element will be missing from output. 
+    # original element will be missing from output.
     #
     # If an input maps to an array, each element of the array will be flattened
-    # into the output. 
+    # into the output.
     #
     # If an input maps to nil, it will cause the input element to be removed
-    # entirely. 
+    # entirely.
     def translate_array(array)
       array.each_with_object([]) do |input_element, output_array|
         output_element = self.map(input_element)
@@ -200,7 +232,7 @@ module Traject
 
     protected
 
-    # No built-in way to read java-style .properties, we hack it. 
+    # No built-in way to read java-style .properties, we hack it.
     # inspired by various hacky things found google ruby java properties parse
     # .properties spec seems to be:
     # http://docs.oracle.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
@@ -213,10 +245,10 @@ module Traject
       f.each_line do |line|
         i += 1
 
-        line.strip! 
+        line.strip!
 
         # skip blank lines
-        next if line.empty? 
+        next if line.empty?
 
         # skip comment lines
         next if line =~ /^\s*[!\#].*$/

data/lib/traject/util.rb

@@ -73,9 +73,9 @@ module Traject
     end
 
     # just does a `require 'java'` but rescues the exception if we
-    # aren't jruby, and raises a better error message.  
+    # aren't jruby, and raises a better error message.
     # Pass in a developer-presentable name of a feature to include in the error
-    # message if you want. 
+    # message if you want.
     def self.jruby_ensure_init!(feature = nil)
       begin
         require 'java'
@@ -89,4 +89,4 @@ module Traject
     end
 
   end
-end
+end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "0.16.0"
+  VERSION = "0.17.0"
 end

data/lib/traject/yaml_writer.rb

@@ -6,4 +6,4 @@ class Traject::YamlWriter < Traject::LineWriter
     context.output_hash.to_yaml(:indentation=>3, :line_width => 78) + "\n\n"
   end
 end
-  
+

data/test/debug_writer_test.rb

@@ -15,11 +15,11 @@ describe 'Simple output' do
     end
     @io = StringIO.new
     @writer = Traject::DebugWriter.new("output_stream" => @io)
-    
+
     @id = "2710183"
     @title = "Manufacturing consent : the political economy of the mass media /"
   end
-  
+
   it "does a simple output" do
     @writer.put Traject::Indexer::Context.new(:output_hash => @indexer.map_record(@record))
     expected = [
@@ -29,10 +29,10 @@ describe 'Simple output' do
     ]
     assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
     @writer.close
-    
+
   end
-  
+
 end
-    
-    
-    
+
+
+

data/test/indexer/each_record_test.rb

@@ -42,18 +42,18 @@ describe "Traject::Indexer#each_record" do
         flunk("Should only fail with a ArityError")
       end
     end
-    
+
     it "rejects each_record with a name (e.g., using a to_field syntax)" do
       assert_raises(Traject::Indexer::NamingError) do
         @indexer.each_record('bad_name') {|one, two| }
       end
     end
-    
+
     it "reject each_record with no arguments/blocks at all" do
       assert_raises(ArgumentError) do
         @indexer.each_record()
       end
-    end    
+    end
 
   end
-end
+end

data/test/indexer/macros_marc21_semantics_test.rb

@@ -198,7 +198,7 @@ describe "Traject::Macros::Marc21Semantics" do
       @record = MARC::Reader.new(support_file_path  "multi_geo.marc").to_a.first
       output = @indexer.map_record(@record)
 
-      assert_equal ["Europe", "Middle East", "Africa, North", "Agora (Athens, Greece)", "Rome (Italy)", "Italy"], 
+      assert_equal ["Europe", "Middle East", "Africa, North", "Agora (Athens, Greece)", "Rome (Italy)", "Italy"],
         output["geo_facet"]
     end
     it "maps nothing on a record with no geo" do
@@ -221,12 +221,12 @@ describe "Traject::Macros::Marc21Semantics" do
     end
 
   end
-  
+
   describe "extract_marc_filing_version" do
     before do
       @record = MARC::Reader.new(support_file_path  "the_business_ren.marc").to_a.first
     end
-    
+
     it "works as expected" do
       @indexer.instance_eval do
         to_field 'title_phrase', extract_marc_filing_version('245ab')
@@ -234,7 +234,7 @@ describe "Traject::Macros::Marc21Semantics" do
       output = @indexer.map_record(@record)
       assert_equal ['Business renaissance quarterly'], output['title_phrase']
     end
-    
+
     it "works with :include_original" do
       @indexer.instance_eval do
         to_field 'title_phrase', extract_marc_filing_version('245ab', :include_original=>true)
@@ -242,7 +242,7 @@ describe "Traject::Macros::Marc21Semantics" do
       output = @indexer.map_record(@record)
       assert_equal ['The Business renaissance quarterly', 'Business renaissance quarterly'], output['title_phrase']
     end
-    
+
     it "doesn't do anything if you don't include the first subfield" do
       @indexer.instance_eval do
         to_field 'title_phrase', extract_marc_filing_version('245h')
@@ -250,8 +250,8 @@ describe "Traject::Macros::Marc21Semantics" do
       output = @indexer.map_record(@record)
       assert_equal ['[electronic resource].'], output['title_phrase']
     end
-      
-    
+
+
     it "dies if you pass it something else" do
       assert_raises(RuntimeError) do
         @indexer.instance_eval do
@@ -259,10 +259,10 @@ describe "Traject::Macros::Marc21Semantics" do
         end
       end
     end
-      
-    
+
+
   end
-      
-      
 
-end
+
+
+end

data/test/indexer/macros_marc21_test.rb

@@ -56,22 +56,22 @@ describe "Traject::Macros::Marc21" do
 
       assert_equal ["DEFAULT VALUE"], output["only_default"]
     end
-    
+
     it "de-duplicates by default, respects :allow_duplicates" do
       # Add a second 008
       f = @record.fields('008').first
       @record.append(f)
-      
+
       @indexer.instance_eval do
         to_field "lang1", extract_marc('008[35-37]')
-        to_field "lang2", extract_marc('008[35-37]', :allow_duplicates=>true)        
+        to_field "lang2", extract_marc('008[35-37]', :allow_duplicates=>true)
       end
-      
+
       output = @indexer.map_record(@record)
       assert_equal ["eng"], output['lang1']
-      assert_equal ["eng", "eng"], output['lang2']      
+      assert_equal ["eng", "eng"], output['lang2']
     end
-    
+
     it "fails on an extra/misspelled argument to extract_marc" do
       assert_raises(RuntimeError) do
         @indexer.instance_eval do
@@ -79,9 +79,9 @@ describe "Traject::Macros::Marc21" do
         end
       end
     end
-      
-    
-      
+
+
+
 
     it "Marc21::trim_punctuation class method" do
       assert_equal "one two three", Marc21.trim_punctuation("one two three")
@@ -177,4 +177,4 @@ describe "Traject::Macros::Marc21" do
   end
 
 
-end
+end

data/test/indexer/macros_test.rb

@@ -37,4 +37,4 @@ describe "Indexer Macros:" do
 
 
 
-end
+end

data/test/indexer/map_record_test.rb

@@ -184,21 +184,21 @@ describe "Traject::Indexer#map_record" do
 
       assert called, "Called mapping routine"
     end
-    
+
     it "skips records" do
-      
+
       @indexer.to_field("beforeSkip") do |rec, acc|
         acc << "Before"
       end
-      
+
       @indexer.to_field('radical') do |rec, acc, context|
         context.skip!("Chomsky!") if rec['245'].to_s =~ /Chomsky/
       end
-      
+
       @indexer.to_field('afterSkip') do |rec, acc|
         acc << "After. Should never happen"
       end
-      
+
       output = @indexer.map_record(@record)
       assert_equal ['Before'], output['beforeSkip']
       assert_nil output['afterSkip']
@@ -206,4 +206,4 @@ describe "Traject::Indexer#map_record" do
 
   end
 
-end
+end

data/test/indexer/read_write_test.rb

@@ -27,9 +27,9 @@ describe "Traject::Indexer#process" do
   end
 
   it "works" do
-    # oops, this times_called counter isn't thread-safe under multi-threading 
+    # oops, this times_called counter isn't thread-safe under multi-threading
     # is why this fails sometimes.
-    # fixed to be single-threaded for these tests. 
+    # fixed to be single-threaded for these tests.
     times_called = 0
     @indexer.to_field("title") do |record, accumulator, context|
       times_called += 1
@@ -68,7 +68,7 @@ describe "Traject::Indexer#process" do
       "solr.url" => "http://example.org",
       "writer_class_name" => "Traject::SolrJWriter"
     )
-    @file = File.open(support_file_path "manufacturing_consent.marc")    
+    @file = File.open(support_file_path "manufacturing_consent.marc")
 
 
     @indexer.to_field("id") do |record, accumulator|
@@ -80,5 +80,44 @@ describe "Traject::Indexer#process" do
     assert ! return_value, "returns false on skipped record errors"
   end
 
+  require 'traject/null_writer'
+  it "calls after_processing after processing" do
+    @indexer = Traject::Indexer.new(
+      "solrj_writer.server_class_name" => "MockSolrServer",
+      "solr.url" => "http://example.org",
+      "writer_class_name" => "Traject::NullWriter"
+    )
+    @file = File.open(support_file_path "test_data.utf8.mrc")
+
+    called = []
+
+    @indexer.after_processing do
+      called << :one
+    end
+    @indexer.after_processing do
+      called << :two
+    end
+
+    @indexer.process(@file)
+
+    assert_equal [:one, :two], called, "Both after_processing hooks called, in order"
+  end
+
+  describe "demo_config.rb" do
+    before do
+      @indexer = Traject::Indexer.new(
+        "solrj_writer.server_class_name" => "MockSolrServer",
+        "solr.url" => "http://example.org",
+        "writer_class_name" => "Traject::NullWriter"
+      )
+    end
+
+    it "parses and loads" do
+      conf_path = support_file_path "demo_config.rb"
+      File.open(conf_path) do |file_io|
+        @indexer.instance_eval(file_io.read, conf_path)
+      end
+    end
+  end
 
-end
+end