traject 0.12.0 → 0.13.0

data/LICENSE.txt

@@ -1,5 +1,3 @@
-Copyright (c) 2013 TODO: Write your name
-
 MIT License
 
 Permission is hereby granted, free of charge, to any person obtaining

data/lib/traject/indexer.rb

@@ -205,6 +205,8 @@ class Traject::Indexer
   # Returns the context passed in as second arg, as a convenience for chaining etc.
   def map_to_context!(context)
     @index_steps.each do |index_step|
+      # Don't bother if we're skipping this record
+      break if context.skip?
       if index_step[:type] == :to_field
 
         accumulator = []
@@ -327,7 +329,12 @@ class Traject::Indexer
       thread_pool.maybe_in_thread_pool do
         context = Context.new(:source_record => record, :settings => settings, :position => position)
         map_to_context!(context)
-        writer.put context
+        if context.skip?
+          log_skip(context)
+        else
+          writer.put context
+        end
+          
       end
 
     end
@@ -353,6 +360,12 @@ class Traject::Indexer
 
     return true
   end
+  
+  # Log that the current record is being skipped, using
+  # data in context.position and context.skipmessage
+  def log_skip(context)
+    logger.debug "Skipped record #{context.position}: #{context.skipmessage}"
+  end
 
   def reader_class
     unless defined? @reader_class
@@ -471,11 +484,29 @@ class Traject::Indexer
       hash_init.each_pair do |key, value|
         self.send("#{key}=", value)
       end
+      
+      @skip = false
     end
 
     attr_accessor :clipboard, :output_hash
     attr_accessor :field_name, :source_record, :settings
     # 1-based position in stream of processed records.
     attr_accessor :position
+    
+    # Should we be skipping this record?
+    attr_accessor :skipmessage
+    
+    # Set the fact that this record should be skipped, with an
+    # optional message
+    def skip!(msg = '(no message given)')
+      @skipmessage = msg
+      @skip = true
+    end
+    
+    # Should we skip this record?
+    def skip?
+      @skip
+    end
+    
   end
 end

data/lib/traject/marc4j_reader.rb

@@ -1,5 +1,6 @@
 require 'traject'
 require 'marc'
+require 'marc/marc4j'
 
 # Uses Marc4J to read the marc records, but then translates them to
 # ruby-marc before delivering them still, Marc4J is just inside the black
@@ -50,28 +51,25 @@ class Traject::Marc4JReader
     @settings     = Traject::Indexer::Settings.new settings
     @input_stream = input_stream
 
-    ensure_marc4j_loaded!
-
     if @settings['marc4j_reader.keep_marc4j'] &&
       ! (MARC::Record.instance_methods.include?(:original_marc4j) &&
          MARC::Record.instance_methods.include?(:"original_marc4j="))
       MARC::Record.class_eval('attr_accessor :original_marc4j')
     end
+    
+    # Creating a converter will do the following:
+    #  - nothing, if it detects that the marc4j jar is already loaded
+    #  - load all the .jar files in settings['marc4j_reader.jar_dir'] if set
+    #  - load the marc4j jar file bundled with MARC::MARC4J otherwise
+     
+    @converter = MARC::MARC4J.new(:jardir => settings['marc4j_reader.jar_dir'], :logger => logger)
+    
+    # Convenience
+    java_import org.marc4j.MarcPermissiveStreamReader
+    java_import org.marc4j.MarcXmlReader
 
   end
 
-  # Loads solrj unless it appears to already be loaded.
-  #
-  # Will load from settings['marc4j_reader.jar_dir'] if given, otherwise
-  # bundled vendor location.
-  #
-  # Will java_import MarcPermissiveStreamReader and MarcXmlReader so you
-  # have those available as un-namespaced classes. 
-  def ensure_marc4j_loaded!
-    unless defined?(MarcPermissiveStreamReader) && defined?(MarcXmlReader)
-      Traject::Util.require_marc4j_jars(settings)
-    end
-  end
 
   def internal_reader
     @internal_reader ||= create_marc_reader!
@@ -101,7 +99,7 @@ class Traject::Marc4JReader
     while (internal_reader.hasNext)
       begin
         marc4j = internal_reader.next
-        rubymarc = convert_marc4j_to_rubymarc(marc4j)
+        rubymarc = @converter.marc4j_to_rubymarc(marc4j)
         if @settings['marc4j_reader.keep_marc4j']
           rubymarc.original_marc4j = marc4j
         end
@@ -123,35 +121,4 @@ class Traject::Marc4JReader
     @logger ||= (settings[:logger] || Yell.new(STDERR, :level => "gt.fatal")) # null logger)
   end
 
-  def convert_marc4j_to_rubymarc(marc4j)
-    rmarc = MARC::Record.new
-    rmarc.leader = marc4j.getLeader.marshal
-
-    marc4j.getControlFields.each do |marc4j_control|
-      rmarc.append( MARC::ControlField.new(marc4j_control.getTag(), marc4j_control.getData )  )
-    end
-
-    marc4j.getDataFields.each do |marc4j_data|
-      rdata = MARC::DataField.new(  marc4j_data.getTag,  marc4j_data.getIndicator1.chr, marc4j_data.getIndicator2.chr )
-
-      marc4j_data.getSubfields.each do |subfield|
-
-        # We assume Marc21, skip corrupted data
-        # if subfield.getCode is more than 255, subsequent .chr
-        # would raise.
-        if subfield.getCode > 255
-          logger.warn("Marc4JReader: Corrupted MARC data, record id #{marc4j.getControlNumber}, field #{marc4j_data.tag}, corrupt subfield code byte #{subfield.getCode}. Skipping subfield, but continuing with record.")
-          next
-        end
-
-        rsubfield = MARC::Subfield.new(subfield.getCode.chr, subfield.getData)
-        rdata.append rsubfield
-      end
-
-      rmarc.append rdata
-    end
-
-    return rmarc
-  end
-
 end

data/lib/traject/solrj_writer.rb

@@ -148,10 +148,7 @@ class Traject::SolrJWriter
 
     if settings["solrj_writer.batch_size"].to_i > 1
       ready_batch = []
-
-      # Synchronize access to our shared batched_queue state,
-      # but once we've pulled out what we want in local var
-      # `ready_batch`, don't need to synchronize anymore.
+      
       batched_queue.add(package)
       if batched_queue.size >= settings["solrj_writer.batch_size"].to_i
         batched_queue.drain_to(ready_batch)

data/lib/traject/util.rb

@@ -26,50 +26,6 @@ module Traject
       str.split(':in `').first
     end
 
-    # Requires marc4j jar(s) from settings['marc4j.jar_dir'] if given, otherwise
-    # uses jars bundled with traject gem in ./vendor
-    #
-    # Have to pass in a settings arg, so we can check it for specified jar dir.
-    #
-    # Tries not to do the dirglob and require if marc4j has already been loaded.
-    # Will define global constants with classes MarcPermissiveStreamReader and MarcXmlReader
-    # if not already defined.
-    #
-    # This is all a bit janky, maybe there's a better way to do this? We do want
-    # a 'require' method defined somewhere utility, so multiple classes can
-    # use it, including extra gems. This method IS used by extra gems, so should
-    # be considered part of the API -- after it's called, those top-level
-    # globals should be available, and marc4j should be loaded.
-    def self.require_marc4j_jars(settings)
-      jruby_ensure_init!
-
-      tries = 0
-      begin
-        tries += 1
-
-        org.marc4j
-
-        # java_import which we'd normally use weirdly doesn't work
-        # from a class method. https://github.com/jruby/jruby/issues/975
-        Object.const_set("MarcPermissiveStreamReader",  org.marc4j.MarcPermissiveStreamReader) unless defined? ::MarcPermissiveStreamReader
-        Object.const_set("MarcXmlReader",  org.marc4j.MarcXmlReader) unless defined? ::MarcXmlReader
-      rescue NameError  => e
-        # /Users/jrochkind/code/solrj-gem/lib"
-
-        include_jar_dir = File.expand_path("../../vendor/marc4j/lib", File.dirname(__FILE__))
-
-        jardir = settings["marc4j.jar_dir"] || include_jar_dir
-        Dir.glob("#{jardir}/*.jar") do |x|
-          require x
-        end
-
-        if tries > 1
-          raise LoadError.new("Can not find Marc4J java classes")
-        else
-          retry
-        end
-      end
-    end
 
     # Requires solrj jar(s) from settings['solrj.jar_dir'] if given, otherwise
     # uses jars bundled with traject gem in ./vendor

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

data/test/indexer/map_record_test.rb

@@ -184,6 +184,25 @@ describe "Traject::Indexer#map_record" do
 
       assert called, "Called mapping routine"
     end
+    
+    it "skips records" do
+      
+      @indexer.to_field("beforeSkip") do |rec, acc|
+        acc << "Before"
+      end
+      
+      @indexer.to_field('radical') do |rec, acc, context|
+        context.skip!("Chomsky!") if rec['245'].to_s =~ /Chomsky/
+      end
+      
+      @indexer.to_field('afterSkip') do |rec, acc|
+        acc << "After. Should never happen"
+      end
+      
+      output = @indexer.map_record(@record)
+      assert_equal ['Before'], output['beforeSkip']
+      assert_nil output['afterSkip']
+    end
 
   end
 

data/traject.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
 
 
   spec.add_dependency "marc", ">= 0.7.1"
+  spec.add_dependency "marc-marc4j", ">=0.1.1"
   spec.add_dependency "hashie", ">= 2.0.5", "< 2.1" # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0"   # command line parsing
   spec.add_dependency "yell" # logging

metadata

@@ -2,14 +2,14 @@
 name: traject
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-09-10 00:00:00.000000000 Z
+date: 2013-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: marc
@@ -27,6 +27,22 @@ dependencies:
     none: false
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: marc-marc4j
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+    none: false
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: hashie
   version_requirements: !ruby/object:Gem::Requirement
@@ -236,8 +252,6 @@ files:
 - test/translation_maps/translate_array_test.yaml
 - test/translation_maps/yaml_map.yaml
 - traject.gemspec
-- vendor/marc4j/README.md
-- vendor/marc4j/lib/marc4j-2.5.1-beta.jar
 - vendor/solrj/README
 - vendor/solrj/build.xml
 - vendor/solrj/ivy.xml

data/vendor/marc4j/README.md

@@ -1,17 +0,0 @@
-The marc4j.jar file that is in `./lib` was created by:
-
-Checking out the marc4j source from github.com/marc4j/marc4j, and building
-with `ant` on 23 July 2013.
-
-That produced the jar you see, with the name it has. (I am not sure the version
-in the jar name is accurate, it's not actually the beta1 release, but the RC1
-release, or maybe even a subsequent release?)
-
-It can be regenerated by doing the same.
-
-Bundling the marc4j jar with our traject gem is not neccesarily the
-best way to go, but it's what we're doing now. See top-level README
-TODO.
-
-(You can use your own custom marc4j.jar by using a runtime setting,
-see Marc4JReader class docs. )