rwdgutenberg 0.12 → 0.13

data/lib/marc/dublincore.rb

@@ -0,0 +1,79 @@
+module MARC
+  
+  # A class for mapping MARC records to Dublin Core
+  
+  class DublinCore
+
+    def self.map(record)
+      dc_hash = Hash.new
+      dc_hash['title'] = get_field_value(record['245']['a'])
+
+      # Creator
+      [100, 110, 111, 700, 710, 711, 720].each do |field|
+        dc_hash['creator'] ||= []
+        dc_hash['creator'] << get_field_value(record[field.to_s])
+      end
+
+      # Subject
+      [600, 610, 611, 630, 650, 653].each do |field|
+        dc_hash['subject'] ||= []
+        dc_hash['subject'] << get_field_value(record[field.to_s])
+      end
+
+      # Description
+      [500..599].each do |field|
+        next if [506, 530, 540, 546].include?(field)
+        dc_hash['description'] ||= []
+        dc_hash['description'] << get_field_value(record[field.to_s])
+      end
+
+      dc_hash['publisher'] = get_field_value(record['260']['a']['b']) rescue nil
+      dc_hash['date'] = get_field_value(record['260']['c']) rescue nil
+      dc_hash['type'] = get_field_value(record['655']) 
+      dc_hash['format'] = get_field_value(record['856']['q']) rescue nil
+      dc_hash['identifier'] = get_field_value(record['856']['u']) rescue nil
+      dc_hash['source'] = get_field_value(record['786']['o']['t']) rescue nil
+      dc_hash['language'] = get_field_value(record['546'])
+
+      dc_hash['relation'] = []
+      dc_hash['relation'] << get_field_value(record['530'])
+      [760..787].each do |field|
+        dc_hash['relation'] << get_field_value(record[field.to_s]['o']['t']) rescue nil
+      end
+
+      [651, 752].each do |field|
+        dc_hash['coverage'] ||= []
+        dc_hash['coverage'] << get_field_value(record[field.to_s])
+      end
+
+      [506, 540].each do |field|
+        dc_hash['rights'] ||= []
+        dc_hash['rights'] << get_field_value(record[field.to_s])
+      end
+      
+      dc_hash.keys.each do |key| 
+        dc_hash[key].flatten! if dc_hash[key].respond_to?(:flatten!)
+        dc_hash[key].compact! if dc_hash[key].respond_to?(:compact!)
+      end
+      
+      dc_hash
+    end
+      
+    def self.get_field_value(field)
+      return if field.nil?
+      
+      if !field.kind_of?(String) && field.respond_to?(:each)
+        values = []
+        field.each do |element|
+          values << get_field_value(element)
+        end
+        values
+      else
+        return field if field.kind_of?(String)
+        return field.value if field.respond_to?(:value)
+      end
+    end
+    
+  end
+end
+

data/lib/marc/exception.rb

@@ -0,0 +1,9 @@
+module MARC
+
+  # basic exception class for exceptions that
+  # can occur during MARC processing.
+
+  class Exception < RuntimeError
+  end
+
+end

data/lib/marc/reader.rb

@@ -0,0 +1,186 @@
+module MARC
+
+  class Reader
+    include Enumerable
+
+    # The constructor which you may pass either a path 
+    #
+    #   reader = MARC::Reader.new('marc.dat')
+    # 
+    # or, if it's more convenient a File object:
+    #
+    #   fh = File.new('marc.dat')
+    #   reader = MARC::Reader.new(fh)
+    #
+    # or really any object that responds to read(n)
+    #
+    #   # marc is a string with a bunch of records in it
+    #   reader = MARC::Reader.new(StringIO.new(reader))
+    
+    def initialize(file)
+      if file.class == String:
+        @handle = File.new(file)
+      elsif file.respond_to?("read", 5)
+        @handle = file
+      else
+        throw "must pass in path or file"
+      end
+    end
+
+    # to support iteration:
+    #   for record in reader
+    #     print record
+    #   end
+    #
+    # and even searching:
+    #   record.find { |f| f['245'] =~ /Huckleberry/ }
+
+    def each 
+      # while there is data left in the file
+      while rec_length_s = @handle.read(5)
+        # make sure the record length looks like an integer
+        rec_length_i = rec_length_s.to_i
+        if rec_length_i == 0:
+          raise MARC::Exception.new("invalid record length: #{rec_length_s}")
+        end
+
+        # get the raw MARC21 for a record back from the file
+        # using the record length
+        raw = rec_length_s + @handle.read(rec_length_i-5)
+        
+
+        # create a record from the data and return it
+        #record = MARC::Record.new_from_marc(raw)
+        record = MARC::Reader.decode(raw)
+        yield record 
+      end
+    end
+
+
+    # A static method for turning raw MARC data in transission
+    # format into a MARC::Record object.
+
+    def self.decode(marc, params={})
+      record = Record.new()
+      record.leader = marc[0..LEADER_LENGTH-1]
+
+      # where the field data starts
+      base_address = record.leader[12..16].to_i
+
+      # get the byte offsets from the record directory
+      directory = marc[LEADER_LENGTH..base_address-1]
+
+      throw "invalid directory in record" if directory == nil
+
+      # the number of fields in the record corresponds to 
+      # how many directory entries there are
+      num_fields = directory.length / DIRECTORY_ENTRY_LENGTH
+
+      # when operating in forgiving mode we just split on end of
+      # field instead of using calculated byte offsets from the 
+      # directory
+      all_fields = marc[base_address..-1].split(END_OF_FIELD)
+
+      0.upto(num_fields-1) do |field_num|
+
+        # pull the directory entry for a field out
+        entry_start = field_num * DIRECTORY_ENTRY_LENGTH
+        entry_end = entry_start + DIRECTORY_ENTRY_LENGTH
+        entry = directory[entry_start..entry_end]
+        
+        # extract the tag
+        tag = entry[0..2]
+
+        # get the actual field data
+        # if we were told to be forgiving we just use the
+        # next available chuck of field data that we 
+        # split apart based on the END_OF_FIELD
+        field_data = ''
+        if params[:forgiving]
+          field_data = all_fields.shift()
+
+        # otherwise we actually use the byte offsets in 
+        # directory to figure out what field data to extract
+        else
+          length = entry[3..6].to_i
+          offset = entry[7..11].to_i
+          field_start = base_address + offset
+          field_end = field_start + length - 1
+          field_data = marc[field_start..field_end]
+        end
+
+        # remove end of field
+        field_data.delete!(END_OF_FIELD)
+         
+        # add a control field or data field
+        if tag < '010'
+          record.append(MARC::ControlField.new(tag,field_data))
+        else
+          field = MARC::DataField.new(tag)
+
+          # get all subfields
+          subfields = field_data.split(SUBFIELD_INDICATOR)
+
+          # must have at least 2 elements (indicators, and 1 subfield)
+          # TODO some sort of logging?
+          next if subfields.length() < 2
+
+          # get indicators
+          indicators = subfields.shift()
+          field.indicator1 = indicators[0,1]
+          field.indicator2 = indicators[1,1]
+
+          # add each subfield to the field
+          subfields.each() do |data|
+            subfield = MARC::Subfield.new(data[0,1],data[1..-1])
+            field.append(subfield)
+          end
+
+          # add the field to the record
+          record.append(field)
+        end
+      end
+
+      return record
+    end
+  end
+
+
+  # Like Reader ForgivingReader lets you read in a batch of MARC21 records
+  # but it does not use record lengths and field byte offets found in the 
+  # leader and directory. It is not unusual to run across MARC records
+  # which have had their offsets calcualted wrong. In situations like this
+  # the vanilla Reader may fail, and you can try to use ForgivingReader.
+  
+  # The one downside to this is that ForgivingReader will assume that the
+  # order of the fields in the directory is the same as the order of fields
+  # in the field data. Hopefully this will be the case, but it is not 
+  # 100% guranteed which is why the normal behavior of Reader is encouraged.
+
+  class ForgivingReader
+    include Enumerable
+
+    def initialize(file)
+      if file.class == String
+        @handle = File.new(file)
+      elsif file.respond_to?("read", 5)
+        @handle = file
+      else
+        throw "must pass in path or File object"        
+      end
+    end
+
+
+    def each 
+      @handle.each_line(END_OF_RECORD) do |raw| 
+        begin
+          record = MARC::Reader.decode(raw, :forgiving => true)
+          yield record 
+        rescue StandardError => e
+          # caught exception just keep barrelling along
+          # TODO add logging
+        end
+      end
+    end
+  end
+end

data/lib/marc/record.rb

@@ -0,0 +1,147 @@
+module MARC
+
+  # A class that represents an individual MARC record. Every record
+  # is made up of a collection of MARC::DataField objects. 
+  #
+  # MARC::Record mixes in Enumerable to enable access to constituent
+  # DataFields. For example, to return a list of all subject DataFields:
+  #
+  #   record.find_all {|field| field.tag =~ /^6../}  
+  # 
+  # The accessor 'fields' is also an Array of MARC::DataField objects which
+  # the client can access or modifyi if neccesary.
+  #
+  #   record.fields.delete(field)
+  # 
+  # Other accessor attribute: 'leader' for record leader as String
+ 
+  class Record
+    include Enumerable
+
+    # the record fields
+    attr_accessor :fields
+
+    # the record leader
+    attr_accessor :leader
+
+    def initialize
+      @fields = []
+      # leader is 24 bytes
+      @leader = ' ' * 24
+      # leader defaults:
+      # http://www.loc.gov/marc/bibliographic/ecbdldrd.html
+      @leader[10..11] = '22'
+      @leader[20..23] = '4500'
+    end
+
+    # add a field to the record
+    #   record.append(MARC::DataField.new( '100', '2', '0', ['a', 'Fred']))
+
+    def append(field)
+      @fields.push(field)
+    end
+
+    # alias to append
+    
+    def <<(field)
+      append(field)
+    end
+
+    # each() is here to support iterating and searching since MARC::Record
+    # mixes in Enumberable
+    #
+    # iterating through the fields in a record:
+    #   record.each { |f| print f }
+    #
+    # getting the 245
+    #   title = record.find {|f| f.tag == '245'}
+    #
+    # getting all subjects
+    #   subjects = record.find_all {|f| ('600'..'699') === f.tag}
+
+    def each
+      for field in @fields
+        yield field
+      end
+    end
+
+    # You can lookup fields using this shorthand:
+    #   title = record['245']
+
+    def [](tag)
+      return self.find {|f| f.tag == tag}
+    end
+
+    # Factory method for creating a MARC::Record from MARC21 in 
+    # transmission format.
+    #
+    #   record = MARC::Record.new_from_marc(marc21)
+    #
+    # in cases where you might be working with somewhat flawed 
+    # MARC data you may want to use the :forgiving parameter which
+    # will bypass using field byte offsets and simply look for the
+    # end of field byte to figure out the end of fields.
+    #
+    #  record = MARC::Record.new_from_marc(marc21, :forgiving => true)
+
+    def self.new_from_marc(raw, params={})
+      return MARC::Reader.decode(raw, params)
+    end
+
+
+    # Returns a record in MARC21 transmission format (ANSI Z39.2). 
+    # Really this is just a wrapper around MARC::MARC21::encode
+    #
+    #   marc = record.to_marc()
+
+    def to_marc 
+      return MARC::Writer.encode(self)
+    end
+
+    # Handy method for returning the MARCXML serialization for a
+    # MARC::Record object. You'll get back a REXML::Document object.
+    # Really this is just a wrapper around MARC::XMLWriter::encode
+    #
+    #   xml_doc = record.to_xml()
+
+    def to_xml
+      return MARC::XMLWriter.encode(self, :include_namespace => true)
+    end
+
+    # Handy method for returning a hash mapping this records values
+    # to the Dublin Core.
+    #
+    #   dc = record.to_dublin_core()
+    #   print dc['title']
+
+    def to_dublin_core
+      return MARC::DublinCore.map(self)
+    end
+
+    # Returns a string version of the record, suitable for printing
+
+    def to_s
+      str = "LEADER #{leader}\n"
+      for field in fields:
+        str += field.to_s() + "\n"
+      end
+      return str
+    end
+
+
+    # For testing if two records can be considered equal.
+
+    def ==(other)
+      return self.to_s == other.to_s
+    end
+
+
+    # Handy for using a record in a regex:
+    #   if record =~ /Gravity's Rainbow/ then print "Slothrop" end
+
+    def =~(regex)
+      return self.to_s =~ regex 
+    end
+
+  end
+end

data/lib/marc/subfield.rb

@@ -0,0 +1,31 @@
+module MARC
+  
+  # A class that represents an individual  subfield within a DataField. 
+  # Accessor attributes include: code (letter subfield code) and value 
+  # (the content of the subfield). Both can be empty string, but should 
+  # not be set to nil. 
+
+  class Subfield
+    attr_accessor :code, :value
+
+    def initialize(code='' ,value='')
+      # can't allow code of value to be nil
+      # or else it'll screw us up later on
+      @code = code == nil ? '' : code
+      @value = value == nil ? '' : value
+    end
+
+    def ==(other)
+      if @code != other.code
+        return false
+      elsif @value != other.value
+        return false
+      end
+      return true
+    end
+
+    def to_s
+      return "$#{code} #{value} "
+    end
+  end
+end