traject 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31243d453f43fbc8f8634c2511340d8fb96c606f
-  data.tar.gz: a843a583f235920931304fc6baa7ff75ce91dfab
+  metadata.gz: 4ae9c6a2d87868021cae1b48637592238387d8a1
+  data.tar.gz: 578c645162da3560ff5e01a28cf43b36e82734f8
 SHA512:
-  metadata.gz: 5a0e6f3c695a5fe497f4cadd14fba7a2a10a3ea1ff40e42a6625949227d1eb3f995dc4d0ecb228f5e98c95f737df1c1e0e8089b59e17fcb0fedca8de5208b535
-  data.tar.gz: 3598e91ed10b039c4382d56ddc69939713425c4f059eb27adfddfdc272e25f2c52e8f900a31b94ceda6576d716c2155188dc03d4697156314e71aadf99a92471
+  metadata.gz: e0bf13c4ff3cab492b6be8922ae22e33311f701756910526eb3206522774b1519db07324531ebd2c366d5907854c78bb6cde0d65eeff78258513d37fae1a3a57
+  data.tar.gz: dd251b15afafe2a11cbefe8493e145d9b9b883183b5f884a5b61557046c74b699427fe2766d5df587e27729f6c22052b2f672b756aba8fee33149f6abbcc4f40

data/Gemfile

@@ -8,5 +8,5 @@ group :development do
 end
 
 group :debug do
-  gem "ruby-debug"
+  gem "ruby-debug", :platform => "jruby"
 end

data/doc/indexing_rules.md

@@ -62,7 +62,7 @@ The third optional argument is a
 object. Most of the time you don't need it, but you can use it for 
 some sophisticated functionality, for example using these Context methods:
 
-* `context.clipboard` A hash into which you can stuff values that you want to pass from one indexing step to another. For example, if you go through a bunch of work to query a database and get a result you'll need more than once, stick the results somewhere in the clipboard.
+* `context.clipboard` A hash into which you can stuff values that you want to pass from one indexing step to another. For example, if you go through a bunch of work to query a database and get a result you'll need more than once, stick the results somewhere in the clipboard. This clipboard is record-specific, and won't persist between records. 
 * `context.position` The position of the record in the input file (e.g., was it the first record, seoncd, etc.). Useful for error reporting
 * `context.output_hash` A hash mapping the field names (generally defined in `to_field` calls) to an array of values to be sent to the writer associated with that field. This allows you to modify what goes to the writer without going through a `to_field` call -- you can just set `context.output_hash['myfield'] = ['my', 'values']` and you're set. See below for more examples
 * `context.skip!(msg)` An assertion that this record should be ignored. No more indexing steps will be called, no results will be sent to the writer, and a `debug`-level log message will be written stating that the record was skipped.
@@ -262,4 +262,4 @@ args for `each_record`.
 
 * **Once you call `context.skip!(msg)` no more index steps will be run for that record**. So if you have any cleanup code, you'll need to make sure to call it yourself.
 
-* **By default, `trajcet` indexing runs multi-threaded**. In the current implementation, the indexing steps for one record are *not* split across threads, but different records can be processed simultaneously by more than one thread. That means you need to make sure your code is thread-safe (or always set `processing_thread_pool` to 0). 
+* **By default, `trajcet` indexing runs multi-threaded**. In the current implementation, the indexing steps for one record are *not* split across threads, but different records can be processed simultaneously by more than one thread. That means you need to make sure your code is thread-safe (or always set `processing_thread_pool` to 0). 

data/lib/traject/macros/marc21_semantics.rb

@@ -26,15 +26,23 @@ module Traject::Macros
         accumulator.concat list.uniq if list
       end
     end
+   
     # If a num begins with a known OCLC prefix, return it without the prefix.
     # otherwise nil.
+    #
+    # Allow (OCoLC) and/or ocn/ocm/on 
+    
+    OCLCPAT = /
+      \A\s*
+      (?:(?:\(OCoLC\)) |
+         (?:\(OCoLC\))?(?:(?:ocm)|(?:ocn)|(?:on))
+         )(\d+)
+         /x
+         
     def self.oclcnum_extract(num)
-      stripped = num.gsub(/\A(ocm)|(ocn)|(on)|(\(OCoLC\))/, '')
-      if num != stripped
-        # it had the prefix, which we've now stripped
-        return stripped
+      if OCLCPAT.match(num)
+        return $1
       else
-        # it didn't have the prefix
         return nil
       end
     end

data/lib/traject/translation_map.rb

@@ -1,6 +1,7 @@
 require 'traject'
 
 require 'yaml'
+require 'dot-properties'
 
 
 module Traject
@@ -14,7 +15,8 @@ module Traject
   #
   # What makes it more useful than a stunted hash is it's ability to load
   # the hash definitions from configuration files, either pure ruby,
-  # yaml, or (limited subset of) java .properties file.
+  # yaml, or java .properties file (not all .properties features may
+  # be supported, we use dot-properties gem for reading)
   #
   # traject's `extract_marc` macro allows you to specify a :translation_map=>filename argument
   # that will automatically find and use a translation map on the resulting data:
@@ -197,7 +199,9 @@ module Traject
     # Returns a dup of internal hash, dup so you can modify it
     # if you like.
     def to_hash
-      @hash.dup
+      dup = @hash.dup
+      dup.delete("__default__")
+      dup
     end
 
     # Run every element of an array through this translation map,
@@ -224,6 +228,24 @@ module Traject
       array.replace( self.translate_array(array))
     end
 
+    # Return a new TranslationMap that results from merging argument on top of self. 
+    # Can be useful for taking an existing translation map, but merging a few
+    # overrides on top. 
+    #
+    #     merged_map = TranslationMap.new(something).merge TranslationMap.new(else)
+    #     #...
+    #     merged_map.translate_array(something) # etc
+    #
+    # If a default is set in the second map, it will merge over the first too. 
+    #
+    # You can also pass in a plain hash as an arg, instead of an existing TranslationMap:
+    #
+    #     TranslationMap.new(something).merge("overridden_key" => "value", "a" => "")
+    def merge(other_map)
+      default = other_map.default || self.default 
+      TranslationMap.new(self.to_hash.merge(other_map.to_hash), :default => default)
+    end
+
     class NotFound < Exception
       def initialize(path)
         super("No translation map definition file found at 'translation_maps/#{path}.[rb|yaml|properties]' in load path: #{$LOAD_PATH}")
@@ -232,36 +254,10 @@ module Traject
 
     protected
 
-    # No built-in way to read java-style .properties, we hack it.
-    # inspired by various hacky things found google ruby java properties parse
-    # .properties spec seems to be:
-    # http://docs.oracle.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
-    #
-    # We do NOT handle split lines, don't do that!
-    def self.read_properties(file_name)
-      hash = {}
-      i = 0
-      f = File.open(file_name)
-      f.each_line do |line|
-        i += 1
-
-        line.strip!
-
-        # skip blank lines
-        next if line.empty?
-
-        # skip comment lines
-        next if line =~ /^\s*[!\#].*$/
-
-        if line =~ /\A([^:=]+)[\:\=]\s*(.*)\s*\Z/
-          hash[$1.strip] = $2
-        else
-          raise IOError.new("Can't parse from #{file_name} line #{i}: #{line}")
-        end
-      end
-      f.close
-
-      return hash
+    # We use dot-properties gem for reading .properties files,
+    # return a hash. 
+    def self.read_properties(file_name)   
+      return DotProperties.load(file_name).to_h      
     end
 
   end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/test/indexer/macros_marc21_semantics_test.rb

@@ -31,6 +31,25 @@ describe "Traject::Macros::Marc21Semantics" do
     
     assert_equal({}, @indexer.map_record(empty_record))
   end
+  
+  it "deals with all prefixed OCLC nunbers" do
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', '(OCoLC)ocm111111111']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', '(OCoLC)222222222']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', 'ocm333333333']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', 'ocn444444444']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', '(OCoLC)ocn555555555']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', '(OCoLC)on666666666']))
+    @record.append(MARC::DataField.new('035', ' ', ' ', ['a', '777777777'])) # not OCLC number
+    
+    @indexer.instance_eval do
+      to_field "oclcnum", oclcnum
+    end
+    output = @indexer.map_record(@record)
+
+    assert_equal %w{47971712 111111111 222222222 333333333 444444444 555555555 666666666},  output["oclcnum"]
+  end
+    
+  
 
   it "#marc_series_facet" do
     @record = MARC::Reader.new(support_file_path  "louis_armstrong.marc").to_a.first

data/test/translation_map_test.rb

@@ -109,7 +109,7 @@ describe "TranslationMap" do
 
     assert_equal "DEFAULT LITERAL", map["not in the map"]
   end
-
+  
   it "respects __default__ __passthrough__" do
     map = Traject::TranslationMap.new("default_passthrough")
 
@@ -135,16 +135,82 @@ describe "TranslationMap" do
     assert_equal ["one"], values
   end
 
-  it "#to_hash" do
-    map = Traject::TranslationMap.new("yaml_map")
+  describe "#to_hash" do
+    it "produces a hash" do
+      map = Traject::TranslationMap.new("yaml_map")
+
+      hash = map.to_hash
+
+      assert_kind_of Hash, hash
+
+      assert ! hash.frozen?, "#to_hash result is not frozen"
+
+      refute_same hash, map.to_hash, "each #to_hash result is a copy"
+    end
+
+    it "does not include __default__ key" do
+      map = Traject::TranslationMap.new("default_passthrough")
+
+      refute map.to_hash.has_key?("__default__")
+      assert_nil map.to_hash["__default__"]
+    end
+
+  end
+
+  describe "#merge" do
+    it "merges" do
+      original = Traject::TranslationMap.new("yaml_map")
+      override = Traject::TranslationMap.new("other" => "OVERRIDE", "new" => "NEW")
+
+      merged = original.merge(override)
 
-    hash = map.to_hash
+      assert_equal "value1",    merged["key1"]
+      assert_equal "OVERRIDE",  merged["other"]
+      assert_equal "NEW",       merged["new"]
+    end
+
+    it "passes through default from first map when no default in second" do
+      original = Traject::TranslationMap.new("yaml_map", :default => "DEFAULT_VALUE")
+      override = Traject::TranslationMap.new("other" => "OVERRIDE")
+
+      merged = original.merge(override)
+
+      assert_equal "DEFAULT_VALUE", merged.default
+      assert_equal "DEFAULT_VALUE", merged["SOME_KEY_NOT_MATCHED"]
+    end
+
+    it "passes through default from second map when no default in first" do
+      original = Traject::TranslationMap.new("yaml_map")
+      override = Traject::TranslationMap.new({"other" => "OVERRIDE"}, :default => "DEFAULT_VALUE")
+
+      merged = original.merge(override)
 
-    assert_kind_of Hash, hash
+      assert_equal "DEFAULT_VALUE", merged.default
+      assert_equal "DEFAULT_VALUE", merged["SOME_KEY_NOT_MATCHED"]
+    end
+
+    it "merges second default on top of first" do
+      original = Traject::TranslationMap.new("yaml_map", :default => "DEFAULT_VALUE")
+      override = Traject::TranslationMap.new({"other" => "OVERRIDE"}, :default => "NEW_DEFAULT_VALUE")
 
-    assert ! hash.frozen?, "#to_hash result is not frozen"
+      merged = original.merge(override)
+
+      assert_equal "NEW_DEFAULT_VALUE", merged.default
+      assert_equal "NEW_DEFAULT_VALUE", merged["SOME_KEY_NOT_MATCHED"]
+    end
+
+    it "merges in a plain hash too" do
+      original = Traject::TranslationMap.new("yaml_map")
+      merged   = original.merge(
+        "other" => "OVERRIDE",
+        "new"   => "NEW"
+      )
+
+      assert_equal "value1",    merged["key1"]
+      assert_equal "OVERRIDE",  merged["other"]
+      assert_equal "NEW",       merged["new"]
+    end
 
-    refute_same hash, map.to_hash, "each #to_hash result is a copy"
   end
 
 end

data/traject.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "hashie", ">= 2.0.5", "< 2.1" # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0"   # command line parsing
   spec.add_dependency "yell" # logging
+  spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,38 +9,48 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-11-25 00:00:00.000000000 Z
+date: 2014-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.0
   name: marc
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.1.1
+  name: marc-marc4j
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: marc-marc4j
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.1.1
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 2.0.5
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  name: hashie
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: hashie
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -49,18 +59,18 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.5
+        version: 3.4.5
     - - <
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '4.0'
+  name: slop
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: slop
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -69,72 +79,76 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 3.4.5
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
+  name: yell
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: yell
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
+  name: dot-properties
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: bundler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: rake
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: minitest
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: minitest
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - none@nowhere.org
@@ -288,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.11
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Index MARC to Solr; or generally process source records to hash-like structures