marc 0.0.8 → 0.0.9

data/lib/marc/reader.rb

@@ -12,7 +12,10 @@ module MARC
     #   fh = File.new('marc.dat')
     #   reader = MARC::Reader.new(fh)
     #
-    # or really any object that responds to read(n).
+    # or really any object that responds to read(n)
+    #
+    #   # marc is a string with a bunch of records in it
+    #   reader = MARC::Reader.new(StringIO.new(reader))
     
     def initialize(file)
       if file.class == String:

data/lib/marc/record.rb

@@ -39,7 +39,7 @@ module MARC
     #   title = record.find {|f| f.tag == '245'}
     #
     # getting all subjects
-    #   subjects = record.find_all {|f| ('600'..'699' === f.tag)}
+    #   subjects = record.find_all {|f| ('600'..'699') === f.tag}
 
     def each
       for field in @fields

data/lib/marc/xmlwriter.rb

@@ -1,4 +1,5 @@
 require 'rexml/document'
+require 'rexml/text'
 
 module MARC
   
@@ -8,8 +9,13 @@ module MARC
     
     # the constructor which you must pass a file path
     # or an object that responds to a write message
+    # the second argument is a hash of options, currently
+    # only supporting one option, stylesheet
+    # 
+    # writer = XMLWriter.new 'marc.xml', :stylesheet => 'style.xsl'
+    # writer.write record
     
-    def initialize(file)
+    def initialize(file, opts={})
       if file.class == String
         @fh = File.new(file,"w")
       elsif file.respond_to?('write')
@@ -19,17 +25,22 @@ module MARC
       end
       
       @fh.write("<?xml version='1.0'?>")
-      
+      if opts[:stylesheet]
+        @fh.write(
+          %Q{<?xml-stylesheet type="text/xsl" href="#{opts[:stylesheet]}"?>})
+      end
       @fh.write("<collection xmlns='" + MARC_NS + "' " +
         "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " +
         "xsi:schemaLocation='" + MARC_NS + " " + MARC_XSD + "'>")
+      @fh.write("\n")
     end
     
     
     # write a record to the file or handle
     
     def write(record)
-      @fh.write(MARC::XMLWriter.encode(record).to_s)
+      MARC::XMLWriter.encode(record).write(@fh, 0)
+      @fh.write("\n");
     end
     
     
@@ -40,24 +51,77 @@ module MARC
       @fh.close
     end
 
+
+    # Converts from ISO 8859-1 to UTF-8, normalizes the UTF-8, and puts a
+    # 'clean up marker' in records that have control characters (which are
+    # not valid in XML). This is useful for locating these records once
+    # they are in XML so problems caused by removing the invalid characters
+    # can be fixed by a person.  This (or something in the module) needs to
+    # convert from MARC-8 to UTF-8, but it doesn't do this yet...
+
+    def self.convert_to_utf8(text)
+      cleaned_text = text.gsub(/[\x00-\x1f\x7f-\xff]+/, ' CLEAN_ME_UP ')
+      utf8_text = cleaned_text.unpack('C*').pack('U*')
+      normalized_text = REXML::Text::normalize(utf8_text)
+
+      return normalized_text
+    end
     
     # a static method that accepts a MARC::Record object
-    # and returns a REXML::Document for the XML serialization
+    # and returns a REXML::Document for the XML serialization.
 
     def self.encode(record)
+      singleChar = Regexp.new(/[\da-z ]{1}/)
+      ctrlFieldTag = Regexp.new(/00[1-9A-Za-z]{1}s/)
+      
+      # Right now, this writer handles input from the strict and
+      # lenient MARC readers. Because it can get 'loose' MARC in, it
+      # attempts to do some cleanup on data values that are not valid
+      # MARCXML.
+      
+      # TODO? Perhaps the 'loose MARC' checks should be split out
+      # into a tolerant MARCXMLWriter allowing the main one to skip
+      # this extra work.
+      
+      # TODO: At the very least there should be some logging
+      # to record our attempts to account for less than perfect MARC.
+      
       root = "<record/>"
-      doc = REXML::Document.new root
+      doc = REXML::Document.new(root)
+
+      # MARCXML only allows alphanumerics or spaces in the leader
+      record.leader.gsub!(/[^\w|^\s]/, 'Z')
+      
+      # MARCXML is particular about last four characters; ILSes aren't
+      if (record.leader[20..23] != "4500")
+        record.leader[20..23] = "4500"
+      end
 
-      # MARCXML is particular about this; ILSes aren't
-      record.leader[20..24] = "4500"
+      # MARCXML doesn't like a space here so we need a filler character: Z
+      if (record.leader[6..6] == " ")
+        record.leader[6..6] = "Z"
+      end
       
-      leader = REXML::Element.new "leader"
-      leader.add_text record.leader
-      doc.root.add_element leader
+      leader = REXML::Element.new("leader")
+      leader.add_text(record.leader)
+      doc.root.add_element(leader)
       
       for field in record.fields
         if field.class == MARC::DataField 
-          datafield_elem = REXML::Element.new "datafield"
+          datafield_elem = REXML::Element.new("datafield")
+          
+          # If marc is leniently parsed, we may have some dirty data; using
+          # the 'z' ind1 value should help us locate these later to fix
+          if (field.indicator1.match(singleChar) == nil)
+            field.indicator1 = 'z'
+          end
+          
+          # If marc is leniently parsed, we may have some dirty data; using
+          # the 'z' ind2 value should help us locate these later to fix
+          if (field.indicator2.match(singleChar) == nil)
+            field.indicator2 = 'z'
+          end
+          
           datafield_elem.add_attributes({
             "tag"=>field.tag,
             "ind1"=>field.indicator1,
@@ -65,18 +129,33 @@ module MARC
           })
 
           for subfield in field.subfields
-            subfield_element = REXML::Element.new "subfield"
+            subfield_element = REXML::Element.new("subfield")
+            
+            # If marc is leniently parsed, we may have some dirty data; using
+            # the blank subfield code should help us locate these later to fix
+            if (subfield.code.match(singleChar) == nil)
+              subfield.code = ' '
+            end
+            
             subfield_element.add_attribute("code", subfield.code)
-            subfield_element.add_text subfield.value
-            datafield_elem.add_element subfield_element
+            text = MARC::XMLWriter.convert_to_utf8(subfield.value)
+            subfield_element.add_text(text)
+            datafield_elem.add_element(subfield_element)
           end
           
           doc.root.add_element datafield_elem
         elsif field.class == MARC::ControlField
-          control_element = REXML::Element.new "controlfield"
+          control_element = REXML::Element.new("controlfield")
+          
+          # We need a marker for invalid tag values (we use 000)
+          if (field.tag.match(ctrlFieldTag) == nil)
+            field.tag = "00z"
+          end
+          
           control_element.add_attribute("tag", field.tag)
-          control_element.add_text field.value
-          doc.root.add_element control_element
+          text = MARC::XMLWriter.convert_to_utf8(field.value)
+          control_element.add_text(text)
+          doc.root.add_element(control_element)
         end
       end
       

data/test/{tc_xmlreader.rb → tc_xml.rb}

@@ -1,7 +1,7 @@
 require 'test/unit'
 require 'marc'
 
-class XMLReaderTest < Test::Unit::TestCase
+class XMLTest < Test::Unit::TestCase
 
   def otest_batch
     reader = MARC::XMLReader.new('test/batch.xml')
@@ -20,15 +20,18 @@ class XMLReaderTest < Test::Unit::TestCase
     record1.append MARC::DataField.new('245', '0', '4', 
       ['a', 'The Great Ray Charles'], ['h', '[sound recording].'])
 
-    writer = MARC::XMLWriter.new('test/foo.xml')
+    writer = MARC::XMLWriter.new('test/test.xml', :stylesheet => 'style.xsl')
     writer.write(record1)
     writer.close
 
-    reader = MARC::XMLReader.new('test/foo.xml')
+    xml = File.read('test/test.xml')
+    assert_match /<\?xml-stylesheet type="text\/xsl" href="style.xsl"\?>/, xml
+
+    reader = MARC::XMLReader.new('test/test.xml')
     record2 = reader.entries[0]
     assert_equal(record1, record2)
 
-    #File.unlink('test/foo.xml')
+    File.unlink('test/test.xml')
   end
 end
 

data/test/ts_marc.rb

@@ -11,5 +11,4 @@ require 'test/tc_controlfield'
 require 'test/tc_record'
 require 'test/tc_reader'
 require 'test/tc_writer'
-require 'test/tc_xmlwriter'
-require 'test/tc_xmlreader'
+require 'test/tc_xml'

metadata

@@ -3,11 +3,11 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: marc
 version: !ruby/object:Gem::Version 
-  version: 0.0.8
-date: 2006-01-16 00:00:00 -05:00
+  version: 0.0.9
+date: 2006-03-28 00:00:00 -06:00
 summary: A ruby library for working with Machine Readable Cataloging
 require_paths: 
-- lib
+  - lib
 email: ehs@pobox.com
 homepage: http://www.textualize.com/ruby_marc
 rubyforge_project: 
@@ -18,52 +18,45 @@ bindir: bin
 has_rdoc: true
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
   version: 
 platform: ruby
 signing_key: 
 cert_chain: 
 authors: 
-- Ed Summers
+  - Ed Summers
 files: 
-- lib/marc
-- lib/marc.rb
-- lib/marc/writer.rb
-- lib/marc/subfield.rb
-- lib/marc/datafield.rb
-- lib/marc/record.rb
-- lib/marc/constants.rb
-- lib/marc/xmlreader.rb
-- lib/marc/xmlwriter.rb
-- lib/marc/exception.rb
-- lib/marc/controlfield.rb
-- lib/marc/reader.rb
-- test/batch.dat
-- test/batch.xml
-- test/tc_datafield.rb
-- test/tc_reader.rb
-- test/foo.xml
-- test/tc_writer.rb
-- test/tc_controlfield.rb
-- test/tc_subfield.rb
-- test/one.dat
-- test/tc_xmlreader.rb
-- test/tc_xmlwriter.rb
-- test/tc_record.rb
-- test/ts_marc.rb
+  - lib/marc
+  - lib/marc.rb
+  - lib/marc/constants.rb
+  - lib/marc/controlfield.rb
+  - lib/marc/datafield.rb
+  - lib/marc/exception.rb
+  - lib/marc/reader.rb
+  - lib/marc/record.rb
+  - lib/marc/subfield.rb
+  - lib/marc/writer.rb
+  - lib/marc/xmlreader.rb
+  - lib/marc/xmlwriter.rb
+  - test/batch.dat
+  - test/batch.xml
+  - test/one.dat
+  - test/tc_controlfield.rb
+  - test/tc_datafield.rb
+  - test/tc_reader.rb
+  - test/tc_record.rb
+  - test/tc_subfield.rb
+  - test/tc_writer.rb
+  - test/tc_xml.rb
+  - test/ts_marc.rb
 test_files: 
-- test/ts_marc.rb
+  - test/ts_marc.rb
 rdoc_options: []
-
 extra_rdoc_files: []
-
 executables: []
-
 extensions: []
-
 requirements: []
-
-dependencies: []
-
+dependencies: []

data/test/foo.xml

@@ -1 +0,0 @@
-<?xml version='1.0'?><collection xmlns='http://www.loc.gov/MARC21/slim' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd'><record><leader>00925njm  22002777a 4500</leader><controlfield tag='007'>sdubumennmplu</controlfield><datafield tag='245' ind1='0' ind2='4'><subfield code='a'>The Great Ray Charles</subfield><subfield code='h'>[sound recording].</subfield></datafield></record></collection>

data/test/tc_xmlwriter.rb

@@ -1,37 +0,0 @@
-require 'test/unit'
-require 'marc'
-
-class XMLWriterTest < Test::Unit::TestCase
-
-  def test_writer()
-    # get a record
-    reader = MARC::Reader.new('test/one.dat')
-    record = reader.entries[0]
-
-    str_writer = StringWriter.new()
-    xml_writer = MARC::XMLWriter.new(str_writer)
-    xml_writer.write(record)
-    assert_match /<\?xml version='1.0'\?>/, str_writer.buffer
-  end
-end
-
-# little class that enables wriing to a string
-# like it's a file
-
-class StringWriter
-  attr_reader :buffer
-
-  def initialize
-    @buffer = ''
-  end
-
-  def write(str)
-    @buffer += str
-  end
-
-  def to_s
-    return @buffer
-  end
-end
-
-