traject_horizon 0.10.2 → 0.11.0

data/lib/traject/horizon_reader.rb

@@ -109,14 +109,6 @@ module Traject
   # == Misc
   #
   # [horizon.batch_size] Batch size to use for fetching item/copy info on each bib. Default 400.
-  #
-  # [horizon.thread_pool] Default 2. HorizonReader uses threads to add some concurrency. If
-  #    set to >0, the work of secondary DB queries for item/copy info and the subsequent
-  #    yielding to caller will be done in threads. Playing with this number
-  #    may be able to increase performance -- recommend not setting it to 0
-  #    except maybe for debugging, having it at least 1 significantly improves
-  #    throughput. 
-  # 
   # [debug_ascii_progress]  if true, will output a "<" and a ">" to stderr around every copy/item
   #           subsidiary fetch. See description of this setting in docs/settings.md
   #
@@ -135,12 +127,6 @@ module Traject
       @settings = Traject::Indexer::Settings.new( self.class.default_settings).merge(settings)
 
       require_jars!
-
-      pool_size     = settings["horizon.thread_pool"].to_i
-
-      logger.debug("HorizonReader with thread pool size #{pool_size}")
-
-      @thread_pool  = Traject::ThreadPool.new( pool_size )
     end
 
     # Requires marc4j and jtds, and java_import's some classes.
@@ -293,20 +279,15 @@ module Traject
           # new record! Put old one on batch queue.
           record_batch << record if record
 
-          # Any exceptions from background threads?
-          @thread_pool.raise_collected_exception!
-
           # prepare and yield batch?
           if (record_count % batch_size == 0)
-            batch = record_batch
-            record_batch = []
-
-            @thread_pool.maybe_in_thread_pool(batch) do |batch|
-              enhance_batch!(extra_connection, batch)
-              batch.each do |r|
-                yield r
-              end
+            enhance_batch!(extra_connection, record_batch)
+            record_batch.each do |r|
+              # set current_bib_id for error logging
+              current_bib_id = r['001'].value
+              yield r
             end
+            record_batch.clear
           end
 
           # And start new record we've encountered.
@@ -316,7 +297,6 @@ module Traject
           record.append MARC::ControlField.new("001", bib_id.to_s)
         end
 
-
         tagord      = rs.getInt("tagord");
         tag         = rs.getString("tag")
 
@@ -324,8 +304,6 @@ module Traject
         # plus any of our exclude_tags.
         next if tag.nil? || tag == "" || exclude_tags.include?(tag)
 
-        numeric_tag = tag.to_i if tag =~ /\A\d+\Z/
-
         indicators = rs.getString("indicators")
 
         # a packed byte array could be in various columns, in order of preference...
@@ -333,61 +311,20 @@ module Traject
         # Have to get it as bytes and then convert it to String to avoid JDBC messing
         # up the encoding marc8 grr
         authtext = rs.getBytes("xref_longtext") || rs.getBytes("xref_text")
-        if authtext
-          authtext = String.from_java_bytes(authtext)
-          authtext.force_encoding("binary")
-        end
-
         text     = rs.getBytes("longtext") || rs.getBytes("text")
-        if text
-          text = String.from_java_bytes(text)
-          text.force_encoding("binary")
-        end
-
-        text = Traject::HorizonBibAuthMerge.new(tag, text, authtext).merge!
 
-        next if text.nil? # sometimes there's nothing there, skip it.
-
-        # convert from MARC8 to UTF8 if needed
-        text = convert_text!(text, error_handler)
-
-        if numeric_tag && numeric_tag == 0
-          record.leader = text
+        if tag == "000"
+          record.leader =  String.from_java_bytes text
           fix_leader!(record.leader)
-        elsif numeric_tag && numeric_tag == 1
-          # nothing, we add the 001 ourselves first
-        elsif numeric_tag && numeric_tag < 10
-          # control field
-          record.append MARC::ControlField.new(tag, text )
-        else
-          # data field
-          indicator1 = indicators.slice(0)
-          indicator2 = indicators.slice(1)
-
-          data_field = MARC::DataField.new(  tag,  indicator1, indicator2 )
-          record.append data_field
-
-          subfields  = text.split("\x1F")
-
-          subfields.each do |subfield|
-            next if subfield.empty?
-
-            subfield_code = subfield.slice(0)
-            subfield_text = subfield.slice(1, subfield.length)
-
-            data_field.append MARC::Subfield.new(subfield_code, subfield_text)
-          end
+        elsif tag != "001"
+          # we add an 001 ourselves with bib id in another part of code.
+          record.append build_marc_field!(error_handler, tag, indicators, text, authtext)
         end
       end
+
       # last one
       record_batch << record if record
 
-      logger.debug "HorizonReader: Waiting for threadpool work complete..."
-      @thread_pool.shutdown_and_wait
-      logger.debug "HorizonReader: threadpool work complete."
-      @thread_pool.raise_collected_exception!
-
-
       # yield last batch
       enhance_batch!(extra_connection, record_batch)
       record_batch.each do |r|
@@ -395,10 +332,9 @@ module Traject
       end
       record_batch.clear
 
-
-
     rescue Exception => e
       logger.fatal "HorizonReader, unexpected exception at bib id:#{current_bib_id}: #{Traject::Util.exception_to_log_message(e)}"
+      logger.fatal e.backtrace.join("/n")
       raise e
     ensure
       logger.info("HorizonReader: Closing all JDBC objects...")
@@ -421,6 +357,61 @@ module Traject
       logger.info("HorizonReader: Closed JDBC objects")
     end
 
+    # Returns a DataField or ControlField, can return
+    # nil if determined no field can/should be created.
+    #
+    # Do not call for field '0' (leader) or field 001,
+    # this doesn't handle those, will just return nil.
+    #
+    # First arg is a Marc4J ErrorHandler object, kind of a weird implementation
+    # detail.
+    #
+    # Other args are objects fetched from Horizon db via JDBC --
+    # text and authtext must be byte arrays.
+    def build_marc_field!(error_handler, tag, indicators, text, authtext)
+      # convert text and authtext from java bytes to a ruby
+      # binary string.
+      if text
+        text = String.from_java_bytes(text)
+        text.force_encoding("binary")
+      end
+      if authtext
+        authtext = String.from_java_bytes(authtext)
+        authtext.force_encoding("binary")
+      end
+
+      text = Traject::HorizonBibAuthMerge.new(tag, text, authtext).merge!
+
+      return nil if text.nil? # sometimes there's nothing there, skip it.
+
+      # convert from MARC8 to UTF8 if needed
+      text = convert_text!(text, error_handler)
+
+      if MARC::ControlField.control_tag?(tag)
+        # control field
+        return MARC::ControlField.new(tag, text )
+      else
+        # data field
+        indicator1 = indicators.slice(0)
+        indicator2 = indicators.slice(1)
+
+        data_field = MARC::DataField.new(  tag,  indicator1, indicator2 )
+
+        subfields  = text.split("\x1F")
+
+        subfields.each do |subfield|
+          next if subfield.empty?
+
+          subfield_code = subfield.slice(0)
+          subfield_text = subfield.slice(1, subfield.length)
+
+          data_field.append MARC::Subfield.new(subfield_code, subfield_text)
+        end
+        return data_field
+      end
+
+    end
+
     # Pass in an array of MARC::Records', adds fields for copy and item
     # info if so configured. Returns record_batch so you can chain if you want.
     def enhance_batch!(conn, record_batch)
@@ -528,9 +519,6 @@ module Traject
       # It might be higher performance to refactor to re-use the same prepared statement
       # for each item/copy fetch... but appears to be no great way to do that in JDBC3
       # where you need to parameterize "IN" values. JDBC4 has got it, but jTDS is just JDBC3.
-      #
-      # Also, we need to be thread-safe now, so maybe better to create one
-      # each time, rather than risk sharing between threads accidentally. 
       pstmt = conn.prepareStatement(sql);
       rs = pstmt.executeQuery
 
@@ -613,7 +601,6 @@ module Traject
     def self.default_settings
       {
         "horizon.batch_size" => 400,
-        "horizon.thread_pool" => 2,
 
         "horizon.public_only" => true,
 

data/lib/traject_horizon/version.rb

@@ -1,3 +1,3 @@
 module TrajectHorizon
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end

metadata

@@ -2,14 +2,14 @@
 name: traject_horizon
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-09-10 00:00:00.000000000 Z
+date: 2013-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: traject