marc4j4r 1.1.0 → 1.2.0

data/CHANGES

@@ -1,3 +1,8 @@
+1.2.0
+  Fixed encoding problem with to_marc and from_string roundtrip
+  Added to_hash/to_marc_in_json and from_hash/from_marc_in_json (see 
+  http://dilettantes.code4lib.org/blog/2010/09/a-proposal-to-serialize-marc-in-json/)
+  
 1.1
   Added native java method to turn a record into XML (20% speedup or so)
 1.0

data/Rakefile

@@ -1,6 +1,6 @@
 require 'rubygems'
 require 'rake'
-require 'ftools'
+# require 'ftools'
 
 begin
   require 'jeweler'
@@ -14,6 +14,8 @@ begin
     gem.add_development_dependency "bacon", ">= 0"
     gem.add_development_dependency "yard", ">= 0"
     
+    gem.files.include 'jars/*.jar'
+    
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new
@@ -27,6 +29,7 @@ LOCALJAR = 'jars/javamarc.jar'
 JAVAJAR  = '../../javamarc.jar'
 
 
+
 file LOCALJAR => JAVAJAR do |t|
   File.copy(JAVAJAR, LOCALJAR)
 end

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

data/benchj.rb

@@ -0,0 +1,73 @@
+$:.unshift 'lib'
+require 'marc4j4r'
+
+require 'benchmark'
+
+# require 'yajl'
+# require 'json/pure'
+# require 'msgpack'
+
+
+jsonsize = 0.0
+marcsize = 0.0
+mpsize   = 0.0
+
+
+# Use Benchmark.measure
+# sjptime =  Benchmark::Tms.new(0,0,0,0,0, "JSON Pure")
+smtime =  Benchmark::Tms.new(0,0,0,0,0, "MARC")
+smptime =  Benchmark::Tms.new(0,0,0,0,0, "Msgpack")
+sjptime =  Benchmark::Tms.new(0,0,0,0,0, "JSON")
+
+djptime =  Benchmark::Tms.new(0,0,0,0,0, "JSON")
+dmtime =  Benchmark::Tms.new(0,0,0,0,0, "MARC")
+dmptime =  Benchmark::Tms.new(0,0,0,0,0, "Msgpack")
+
+
+
+i = 0
+iterations = 1
+
+iterations.times do
+  reader = MARC4J4R::Reader.new('topics.xml', :marcxml)
+  
+  reader.each_with_index do |r, i|
+   marc = nil
+   json = nil
+   mp   = nil
+   copy = nil
+ 
+
+   smtime += Benchmark.measure {marc = r.to_marc}
+   dmtime += Benchmark.measure {copy = MARC4J4R::Record.from_string(marc)}
+   
+
+   sjptime += Benchmark.measure {json = r.to_marc_in_json}
+   djptime += Benchmark.measure {copy = MARC4J4R::Record.new_from_marc_in_json(json)}
+   
+   # break if i > 1000
+
+  end
+end
+
+puts "Total of #{i} records run #{iterations} times"
+
+puts "\nSERIALIZING"
+
+base = smtime.total
+puts '  %-15s %8.2f s (%3.0f%%)' % ['MARC', smtime.total, smtime.total / base * 100]
+puts '  %-15s %8.2f s (%3.0f%%)' % ['Json', sjptime.total, sjptime.total / base * 100]
+# puts '  %-15s %8.2f s (%3.0f%%)' % ['Msgpack', smptime.total, smptime.total / base * 100]
+
+base = dmtime.total
+puts "\nDESERIALIZING"
+puts '  %-15s %8.2f s (%3.0f%%)' % ['MARC', dmtime.total, dmtime.total / base * 100]
+puts '  %-15s %8.2f s (%3.0f%%)' % ['Json', djptime.total, djptime.total / base * 100]
+# puts '  %-15s %8.2f s (%3.0f%%)' % ['Msgpack', dmptime.total, dmptime.total / base * 100]
+
+base = dmtime.total + smtime.total
+puts "\nSERIALIZE + DESERIALIZE"
+puts '  %-15s %8.2f s (%3.0f%%)' % ['MARC', dmtime.total + smtime.total, (dmtime.total + smtime.total) / base * 100]
+puts '  %-15s %8.2f s (%3.0f%%)' % ['Json', djptime.total + sjptime.total, (djptime.total + sjptime.total) / base * 100]
+# puts '  %-15s %8.2f s (%3.0f%%)' % ['Msgpack', dmptime.total + smptime.total, (dmptime.total + smptime.total) / base * 100]
+

data/lib/marc4j4r/record.rb

@@ -1,3 +1,4 @@
+require 'stringio'
 module MARC4J4R
   Record =  Java::org.marc4j.marc.impl::RecordImpl
 
@@ -17,10 +18,16 @@ module MARC4J4R
     def == other
       return false unless (self.leader == other.leader)
       self.zip(other) do |so|
-        return false unless so[0] == so[1]
+        unless so[0] == so[1]
+          puts "self <> other\n#{so[0]}\n#{so[1]}"
+          return false;
+        end
       end
       other.zip(self) do |so|
-        return false unless so[0] == so[1]
+        unless so[0] == so[1]
+          puts "#{so[0]}\n#{so[1]}"
+          return false;
+        end
       end      
       return true
     end
@@ -141,6 +148,7 @@ module MARC4J4R
 
 
     # Return the record as valid MARC-XML
+    # @param String encoding The encoding to use
     # @return String A MARC-XML representation of the record, including the XML header
      
     def to_xml
@@ -148,45 +156,66 @@ module MARC4J4R
     end
       
 
-    def to_marc
-      begin
+    def to_marc encoding='UTF-8'
+      # begin
         s = Java::java.io.ByteArrayOutputStream.new
-        writer =  org.marc4j.MarcPermissiveStreamWriter.new(s)
+        writer =  org.marc4j.MarcPermissiveStreamWriter.new(s, encoding)
         writer.write(self)
-        @marcbinary = s.to_string
-        return @marcbinary
-      rescue
-        # "Woops! to_marc failed for record #{self['001'].data}: #{$!}"
-        "Whoops! Failed: #{$!}"
-      end
+        return s.to_string
+        # writer.close
+        # @marcbinary = s.to_string
+        # return @marcbinary
+      # rescue
+      #   # "Woops! to_marc failed for record #{self['001'].data}: #{$!}"
+      #   "Whoops! Failed: #{$!}"
+      # end
     end
-  end
-    
   
-  def to_marchash
-    h = {}
-    h['type'] = 'marc-hash'
-    h['version'] = [1,0]
-    h['leader'] = self.leader
-
-    fields = []
-
-    self.getVariableFields.each do |f|
-      if f.controlField?
-        fields << [f.tag, f.value]
-      else 
-        farray = [f.tag, f.indicator1 || ' ', f.indicator2 || ' ']
-        subs = []
-        f.each do |subfield|
-          subs << [subfield.code, subfield.value]
+    def to_marchash
+      h = {}
+      h['type'] = 'marc-hash'
+      h['version'] = [1,0]
+      h['leader'] = self.leader
+
+      fields = []
+
+      self.getVariableFields.each do |f|
+        if f.controlField?
+          fields << [f.tag, f.value]
+        else 
+          farray = [f.tag, f.indicator1 || ' ', f.indicator2 || ' ']
+          subs = []
+          f.each do |subfield|
+            subs << [subfield.code, subfield.value]
+          end
+          farray.push subs
+          fields << farray
         end
-        farray.push subs
-        fields << farray
       end
+      h['fields'] = fields
+      return h
+    end
+
+    # Turn it into a marc-in-json hashmap. Note that this won't really work
+    # like a ruby hash; you need to know what you're getting, since stuff
+    # like #each won't work.
+    #
+    # Better to just use to_marc_in_json if you want a json string
+
+    def to_hash
+      return Java::org.marc4j.MarcInJSON.record_to_hash(self)
     end
-    h['fields'] = fields
-    return h
+
+
+    # Turn it into a marc-in-json JSON string using Jackson
+    def to_marc_in_json
+      return Java::org.marc4j.MarcInJSON.record_to_marc_in_json(self)
+    end
+  
+  
   end
+    
+  
 
   # Give a marc record in a string, turn it into an object
   # @param String str The record as a MARC binary string
@@ -195,8 +224,10 @@ module MARC4J4R
   # Note that the normal way of defining this class (self.from_string)
   # didn't work; I assume it has something to do with the fact that
   # it's actually jrst aliased to the Java class
-  def Record.from_string str
-    return MARC4J4R::Reader.new(StringIO.new(str)).first
+  def Record.from_string str, encoding=nil
+    s = Java::java.io.ByteArrayInputStream.new(str.to_java_bytes)
+    # return MARC4J4R::Reader.new(StringIO.new(str), :strictmarc, encoding).first
+    return MARC4J4R::Reader.new(s, :strictmarc, encoding).first
   end
   
   
@@ -207,6 +238,16 @@ module MARC4J4R
     return MARC4J4R::Reader.new(StringIO.new(str), :marcxml).first
   end
   
+  def Record.new_from_hash hash
+    return Java::org.marc4j.MarcInJSON.new_from_hash(hash)
+  end
+  
+  def Record.new_from_marc_in_json jsonstring
+    return Java::org.marc4j.MarcInJSON.new_from_marc_in_json(jsonstring)
+  end
+  
+  
+  
 
 end
 

data/lib/marc4j4r.rb

@@ -5,20 +5,30 @@ end
 require 'logger'
 $LOG ||= Logger.new(STDERR)
 
+jardir = File.join(File.dirname(__FILE__), '..', 'jars')
+
+# For each jar, check for a representative class in each
+# and include the jar if it's not defined
+
 begin
   include_class Java::org.marc4j.marc.impl.RecordImpl
 rescue NameError  => e
-  jardir = File.join(File.dirname(__FILE__), '..', 'jars')
   require "#{jardir}/javamarc.jar"
 end
 
 begin
   include_class Java::org.marc4j.MarcAlephSequentialReader
 rescue
-  jardir = File.join(File.dirname(__FILE__), '..', 'jars')
-  require "#{jardir}/marc4j_serializations.jar"
+  require "#{jardir}/marc4j-extra-readers-writers.jar"
 end
 
+begin
+  include_class Java::org.codehaus.jackson.map.ObjectMapper
+rescue
+  require "#{jardir}/jackson-all-1.6.0.jar"
+end
+
+
 # Define a method that will take a string (filename), IO object, or StringIO object,
 # and return an inputstream/outputstream
 

data/spec/record_spec.rb

@@ -48,7 +48,7 @@ describe "MARC4J4R_basic_retrieval_stuff" do
   end
   
   it "can find all the fields" do 
-    fields = @one.collect
+    fields = @one.collect {|a| a}
     fields.size.should.equal 16
   end
   
@@ -70,6 +70,8 @@ describe "MARC4J4R::Record #find_by_tag" do
     
     @cf = MARC4J4R::ControlField.new('005', '20071104155141.9')
     
+    @batch = MARC4J4R::Reader.new("#{DIR}/batch.dat").collect
+    @utf8 = MARC4J4R::Reader.new("#{DIR}/chinese_utf8.dat").first
   end  
 
   it "gets an empty array for non-existant tag(s)" do
@@ -112,14 +114,33 @@ describe "MARC4J4R::Record #find_by_tag" do
     first = @one['700']
     @one.find_by_tag('700')[0].should.equal first
   end
+
+end
+
+describe "Format checks" do
+  before do
+    @one = MARC4J4R::Reader.new("#{DIR}/one.dat").first
+    @batch = MARC4J4R::Reader.new("#{DIR}/batch.dat").collect {|a| a}
+    @utf8 = MARC4J4R::Reader.new("#{DIR}/chinese_utf8.dat").first
+  end  
   
-  it "round trips binary and xml" do
+  it "round trips binary" do
     MARC4J4R::Record.from_string(@one.to_marc).should.equal @one
-    MARC4J4R::Record.from_xml_string(@one.to_xml).should.equal @one
+    @batch.each do |r|
+      MARC4J4R::Record.from_string(r.to_marc).should.equal r
+    end
+  end
+  
+  it "round trips utf8 record as binary" do
+    MARC4J4R::Record.from_string(@utf8.to_marc, :utf8).should.equal @utf8
   end
   
-  it "round trips xml with the native_to_xml" do
-    MARC4J4R::Record.from_xml_string(@one.native_to_xml).should.equal @one
+  it "round trips XML" do
+    MARC4J4R::Record.from_xml_string(@one.to_xml).should.equal @one
   end
     
+  it "round trips marc-in-json" do
+    copy = @one.to_marc_in_json
+    MARC4J4R::Record.new_from_marc_in_json(copy).should.equal @one
+  end  
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: marc4j4r
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 31
   prerelease: false
   segments: 
   - 1
-  - 1
+  - 2
   - 0
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors: 
 - BillDueber
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-16 00:00:00 -04:00
+date: 2010-10-05 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -60,8 +60,10 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- benchj.rb
+- jars/jackson-all-1.6.0.jar
 - jars/javamarc.jar
-- jars/marc4j_serializations.jar
+- jars/marc4j-extra-readers-writers.jar
 - lib/marc4j4r.rb
 - lib/marc4j4r/controlfield.rb
 - lib/marc4j4r/datafield.rb