traject_horizon 0.11.1 → 0.11.2

data/lib/traject/horizon_reader.rb

@@ -128,6 +128,8 @@ module Traject
       @settings = Traject::Indexer::Settings.new( self.class.default_settings).merge(settings)
 
       require_jars!
+
+      logger.info("   #{self.class.name} reading records from #{settings["horizon.jdbc_url"]}")
     end
 
     # Requires marc4j and jtds, and java_import's some classes.
@@ -210,6 +212,18 @@ module Traject
         sql += " AND " + clauses.join(" AND ") + " "
       end
 
+      # without the order by, rows USUALLY come back in order anyway,
+      # but sometimes they don't -- when they don't, it can cause one real
+      # record to be split up into multiple partial output record, which
+      # cna overwrite each other in the solr index.
+      #
+      # So we sort -- which seems to make query results come back somewhat
+      # slower, but SEEMS to be managagle. Ideally we might include 'tagord'
+      # in the sort too, but that seems to make performance even worse,
+      # we're willing to risk tags not being reassembled in exactly the
+      # right order, usually they are anyway, and it doesn't usually matter anyway.
+      sql+= " ORDER BY b.bib# " # ", tagord" would be even better, but slower.
+
       pstmt = conn.prepareStatement(sql);
 
       # this may be what's neccesary to keep the driver from fetching
@@ -315,6 +329,7 @@ module Traject
         authtext = rs.getBytes("xref_longtext") || rs.getBytes("xref_text")
         text     = rs.getBytes("longtext") || rs.getBytes("text")
 
+
         if tag == "000"
           # Horizon puts a \x1E marc field terminator on the end of hte
           # leader in the db too, but that's not really part of it.
@@ -323,7 +338,8 @@ module Traject
           fix_leader!(record.leader)
         elsif tag != "001"
           # we add an 001 ourselves with bib id in another part of code.
-          record.append build_marc_field!(error_handler, tag, indicators, text, authtext)
+          field = build_marc_field!(error_handler, tag, indicators, text, authtext)
+          record.append field unless field.nil?
         end
       end
 
@@ -332,6 +348,7 @@ module Traject
 
       # yield last batch
       enhance_batch!(extra_connection, record_batch)
+
       record_batch.each do |r|
         yield r
       end
@@ -374,6 +391,7 @@ module Traject
     # Other args are objects fetched from Horizon db via JDBC --
     # text and authtext must be byte arrays.
     def build_marc_field!(error_handler, tag, indicators, text, authtext)
+
       # convert text and authtext from java bytes to a ruby
       # binary string.
       if text
@@ -561,6 +579,7 @@ module Traject
 
     # Mutate string passed in to fix leader bytes for marc21
     def fix_leader!(leader)
+
       if leader.length < 24
         # pad it to 24 bytes, leader is supposed to be 24 bytes
         leader.replace(  leader.ljust(24, ' ')  )
@@ -573,12 +592,26 @@ module Traject
         leader[9] = 'a'
       end
 
-      # Do not understand why this voodoo that should be a no-op is neccesary,
-      # but get a mysterious and hard to isolate/reproduce encoding
-      # bug without it, but not with it. Think it may be the same
-      # but as this:
-      # https://github.com/jruby/jruby/issues/886
-      leader.force_encoding(leader.encoding)
+      # leader should only have ascii chars in it; invalid non-ascii
+      # chars can cause ruby encoding problems down the line.
+      # additionally, a force_encoding may be neccesary to
+      # deal with apparent weird hard to isolate jruby bug prob same one
+      # as at https://github.com/jruby/jruby/issues/886
+      leader.force_encoding('ascii')
+
+      unless leader.valid_encoding?
+        # replace any non-ascii chars with a space.
+
+        # Can't access leader.chars when it's not a valid encoding
+        # without a weird index out of bounds exception, think it's
+        # https://github.com/jruby/jruby/issues/886
+        # Grr.
+
+        #leader.replace( leader.chars.collect { |c| c.valid_encoding? ? c : ' ' }.join('') )
+        leader.replace(leader.split('').collect { |c| c.valid_encoding? ? c : ' ' }.join(''))
+      end
+
+
 
     end
 

data/lib/traject_horizon/version.rb

@@ -1,3 +1,3 @@
 module TrajectHorizon
-  VERSION = "0.11.1"
+  VERSION = "0.11.2"
 end

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: traject_horizon
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 0.11.1
+  version: 0.11.2
+  prerelease: 
 platform: ruby
 authors:
 - Jonathan Rochkind
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-16 00:00:00.000000000 Z
+date: 2013-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: traject
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
   name: marc-marc4j
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
-    none: false
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: minitest
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
-  type: :development
-description:
+description: 
 email:
 - jonathan@dnil.net
 executables: []
@@ -115,26 +115,26 @@ files:
 homepage: http://github.com/jrochkind/traject_horizon
 licenses:
 - MIT
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
-rubyforge_project:
-rubygems_version: 1.8.24
-signing_key:
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
 specification_version: 3
 summary: Horizon ILS MARC Exporter, a plugin for the traject tool
 test_files: