marc_alephsequential 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Yjk3YTZkMTY4Njg0NGY3ZTQ5ZGY2MzQ3MGIxMDA4NTdjMzEzZmNkNA==
+  data.tar.gz: !binary |-
+    NmMwMWQzMTIzMmQ3ZmNlOTM5YjRiMGFmZTI2MTA3NmM0Njk0ZTJiYg==
+SHA512:
+  metadata.gz: !binary |-
+    NWE0M2NiM2M5MTcwOTJjNWJkN2JiMDk5OTJmNWIxYmVmNTg4NGY1ZTAxZGE1
+    OTM4ZDBhMTA3ZDZiYmY0YWU3M2I5NWM5MzBlYjhkY2QxNGZhNDdhMDAyM2Qx
+    MjQ4YWFiNTkxMjhmODk3Mzc0YjIzMGVlMzVmYzRlOTc0Y2Y3YmE=
+  data.tar.gz: !binary |-
+    OWZmOGVkYWVkZTM4ZGQ1YmM1NGMzNzBkM2IwY2I4NmZmMmI1OTYzN2FhMTQw
+    NDcwMDQ0Zjk3MzdiZWQxN2M1ZmQzMGRhYWFkNTg5NGFiNTY4Yjk4N2JiMTFi
+    ZWEyZjJkMThkY2RlM2VjYmZhNTQ1NmYxYzM5NDhlN2QyOGY3MGE=

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+doc/
+pkg/
+vendor/cache/*.gem

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+bundler_args: --without documentation
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+branches:
+  only:
+    - master
+    - /^ci-.*$/ 

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "marc_alephsequential Documentation" --protected

data/ChangeLog.md

@@ -0,0 +1,4 @@
+### 0.1.0 / 2013-08-12
+
+* Initial release:
+

data/Gemfile

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'marc'
+gem 'yell'
+
+group :test do
+  gem 'minitest'
+  gem "minitest-reporters", '>= 0.8.0'
+  gem 'minitest-colorize'
+end
+
+group :development do
+  gem 'kramdown'
+  gem 'bundler', '~> 1'
+  gem 'rake', '>= 1.0'
+  gem 'yard', '>= 0.8'
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Bill Dueber
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,103 @@
+# marc_alephsequential
+[![Build Status](https://secure.travis-ci.org/billdueber/marc_alephsequential.png)](http://travis-ci.org/billdueber/marc_alephsequential)
+
+A [ruby-marc](https://github.com/ruby-marc/ruby-marc) reader for MARC files in the Aleph sequential format
+
+* [Homepage](https://github.com/billdueber/marc_alephsequential#readme)
+* [Issues](https://github.com/billdueber/marc_alephsequential/issues)
+* [Documentation](http://rubydoc.info/gems/marc_alephsequential/frames)
+* [Email](mailto:bill at dueber.com)
+
+## Examples
+
+```ruby
+
+  require 'marc'
+  require 'marc_alephsequential'
+
+  log = GetALogFromSomewhere.new
+  # reader = MARC::AlephSequential::Reader.new('myfile.seq')
+  reader = MARC::AlephSequential::Reader.new('myfile.seq.gz') # automatically notice the .gz and behave!
+  
+  reader.log = log # optional. Set up a logger; otherwise, a default logger will be used
+  
+  begin
+    reader.each do |r|
+      # do stuff with the record
+    end  
+  rescue MARC::AlephSequential::Error => e
+    log.error "Error while parsing record #{e.record_id} at/near #{e.line_number}: #{e.message}"
+    retry # may or may not work the way you'd hope/expect
+  rescue => e
+    log.error "Other error of some sort. quitting. #{e.message}"
+  end
+
+```
+
+## Description of the Aleph Sequential format
+
+Aleph sequential is a MARC serialization format that is easily output by Ex Libris' Aleph software.
+Each MARC record is presented as a series of unicode text lines, one field per line.
+
+
+    000000228 LDR   L ^^^^^nam^a22002891^^4500
+    000000228 001   L 000000228
+    000000228 006   L m^^^^^^^^d^^^^^^^^
+    000000228 007   L cr^bn^---auaua
+    000000228 008   L 880715r19691828nyuab^^^^^^^^|00000^eng^^
+    000000228 010   L $$a68055188
+    000000228 020   L $$a083711750X
+    000000228 035   L $$a(RLIN)MIUG0021856-B
+    000000794 24514 L $$aThe descent of manuscripts.
+    000000794 60010 L $$aCicero, Marcus Tullius$$xManuscripts.
+    000000794 60000 L $$aPlato.$$tCritias$$xManuscripts.
+
+Each line has the following format (note: All must be in utf-8)
+
+* 9 characters (all digits) for the aleph record ID
+* [space]
+* 3 character tag (left-justified / space padded if need be)
+* 1 character indicator 1
+* 1 character indicator 2
+* [space L space], for some historic reasons I don't know
+* The tag's value, perhaps with internal subfields
+
+A record is defined as a set of continuous lines with the same record ID (i.e., the way you know you've finished with a record is because the record ID changes or you hit EOF). 
+
+### How to read the Aleph sequential "value"
+
+The leader and control fields have no internal structure, but spaces in the values are stored as '^' for some reason. (The reader, obviously, changes them back into spaces)
+
+For data fields, the subfields are indicated as follows:
+
+* A _subfield start marker_ (let's just say "SSM") matches /\$\$[a-z0-9]/ (e.g., $$a)
+* The value string for a data field must start with an SSM 
+* An SSM marks the start of a subfield (and the end of the previous subfield, if any)
+
+### Obvious limitations of the Aleph sequential format
+
+Actually, it's not all bad; I like it in a lot of ways. A little verbose at times, but easy to read for a human, and easy to write one-off scripts to run through a file and get statistics about use of tags, find a specific record (just match the bib ID at the beginning of the line), etc. 
+
+The easy-to-see problems are:
+
+* fixed field size. Aleph has a lot of Cobol underneath. So if your bib ids don't happen to be nine characters, well, too bad.
+* You can't have an embedded '$$' in a data field's value, because it will be interpreted as the start of a new subfield. '$$' isn't super common as a typo, but I've seen it.
+
+
+## Parse errors and automatic workarounds
+
+* Lines that don't start with a nine-digit id will be assumed to be a part of the previous line that has an illegal spurious newline. The newline will be removed and all put back together again. If there is no "previous line" because it's the first line of the file, throw an error.
+* Any completed record that doesn't include a leader (LDR) will throw an error
+* Datafield values that don't start with '$$' will be logged as an error and assumed that the first set of data should be in subfield $$a
+
+
+
+## Install
+
+    $ gem install marc_alephsequential
+
+## Copyright
+
+Copyright (c) 2013 Bill Dueber
+
+See {file:LICENSE.txt} for details.

data/Rakefile

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+require 'rubygems'
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install bundler` to install Bundler."
+  exit -1
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn "Run `bundle install` to install missing gems."
+  exit e.status_code
+end
+
+require 'rake'
+
+require "bundler/gem_tasks"
+
+require 'yard'
+YARD::Rake::YardocTask.new  
+task :doc => :yard
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs.push 'lib'
+  t.libs.push 'spec'
+  t.test_files = Dir.glob('spec/**/*_spec.rb')
+end
+
+task :spec => :test
+ 
+task(:default => :test)

data/lib/marc_alephsequential/asline.rb

@@ -0,0 +1,161 @@
+require 'marc'
+require_relative 'error'
+require_relative 'log'
+
+module MARC
+  module AlephSequential
+    
+    
+    #  A model of a line (field) in an alephsequential file.
+    class ASLine
+      
+      include Log
+
+      # Characters in leader/control fields that need to be turned (back) into spaces
+      TURN_TO_SPACE = /\^/
+
+      # Pattern used to split data field values into subfield code/value pairs      
+      SUBFIELD_SPLIT_PATTERN = /\$\$([a-zA-Z0-9])/
+      
+      # How to know if we have a valid id? Must be 9 digits
+      VALID_ID = /^\d{9}$/
+      
+      # The passed in raw string, used for post-processing later on
+      attr_accessor :rawstr
+      
+      # The line number in the file/stream, for error reporting
+      attr_accessor :line_number
+      
+      # Either the value of a control/fiexed field, or a string representation of a datafield's subfield
+      attr_accessor :value
+      
+      # The type of field (:leader, :control, :data, or :invalid_id)
+      attr_accessor :type
+      
+      attr_accessor :id,  :tag, :ind1, :ind2
+
+      # The MARC field's tag
+      attr_reader :tag
+      
+      
+      # Given a raw string and a line number, construct the appropriate ASLine.
+      # 
+      # @param [String] rawstr The raw string from the file 
+      # @param [Number] line_number The line number from the file/stream, for error reporting
+      
+      def initialize(rawstr, line_number)
+        @rawstr = rawstr.chomp
+        @line_number = line_number
+                
+        (self.id,self.tag,self.ind1,self.ind2,self.value) = *(parseline(@rawstr))
+        
+        # clean up the leader or fixed fields
+        if [:leader, :control].include? self.type
+          self.value = cleanup_fixed(self.value)
+        end
+        
+      end
+      
+      # Does this line have a valid (-looking) id? 
+      def valid_id?
+        return VALID_ID.match(id) ? true : false
+      end
+      
+      
+      # Turn it into an actual MARC field (control or data)
+      # Throw an error if called on a leader (LDR) line
+      # @return [MARC::ControlField, MARC::DataField]
+      def to_field
+        case type
+        when :control
+          self.to_control_field
+        when :data
+          self.to_data_field
+        else
+          raise MARC::AlephSequential::Error.new(id, line_number ), "Tried to call #to_field on line type '#{self.type}'", nil
+        end
+      end
+      
+      # Turn the current object into a control field, without doing any checks
+      # @return [MARC::ControlField]
+      def to_control_field
+        MARC::ControlField.new(tag, cleanup_fixed(self.value))
+      end
+      
+      # Turn the current object into a datafield, without doing any checks
+      # @return [MARC::DataField]
+      def to_data_field
+        if self.value[0..1] != '$$'
+          log.error("#{self.line_number} #{self.id} Variable field #{self.tag} doesn't start with '$$'. Prepending '$$a'.")
+          self.value = '$$a' + self.value
+        end
+
+        subfields = parse_string_into_subfields(value)
+        f = MARC::DataField.new(tag, ind1, ind2)
+        f.subfields = subfields
+        return f
+      end  
+      
+      # Parse out a non-controlfield value string into a set of subfields
+      # @param [String] val the value string, of the form "$$athis is the a$$band the b"
+      # @return [Array<Subfield>] An array of MARC subfields
+      #
+      # If the first value in the array returned by the split isn't the empty string, then
+      # the string didn't start with '$$' and we should throw a warning 
+      # (and put the value into a subfield 'a' if we're running in flexible mode)
+      
+      def parse_string_into_subfields(val)
+        sfpairs = val.split(SUBFIELD_SPLIT_PATTERN)
+        initial_null_string = sfpairs.shift
+        unless initial_null_string == ''
+          # do something about the error
+        end
+        
+        sfpairs.each_slice(2).map {|code, val| MARC::Subfield.new(code, val) }
+        
+      end
+      
+      # Clean up fixed fields/leader, turning Ex Libris characters back into normal characters
+      # @param [String] val The string to clean
+      # @return [String] The cleaned string
+      def cleanup_fixed(val)
+        return val.gsub(TURN_TO_SPACE, ' ')
+      end
+        
+      # Set the tag. As a side effect, set the type when we set the tag
+      # type will end up as :leader, :control, :data, or :invalid_id
+      def tag=(t)
+        @tag = t
+        if t == 'LDR'
+          self.type = :leader
+        elsif MARC::ControlField.control_tag?(t)
+          self.type = :control
+        elsif self.valid_id?
+          self.type = :data
+        else
+          self.type = :invalid_id
+        end
+      end
+        
+        
+      # Get a line and parse it out into its componant parts
+      # @param [String] line the line to parse
+      # @return [Array] An array of the form [id, tag, ind1, ind2, value]
+      
+      def parseline(line)
+        id = line[0,9]
+        tag = line[10,3]
+        ind1 = line[13,1]
+        ind2 = line[14,1]
+        value = line[18..-1]
+        return [id,tag,ind1,ind2,value]
+      end
+      
+        
+        
+        
+    end # ASLine
+  end
+end
+    
+ 

data/lib/marc_alephsequential/asline_group.rb

@@ -0,0 +1,101 @@
+require_relative 'asline'
+require_relative 'error'
+require_relative 'log'
+
+
+module MARC
+  module AlephSequential
+    
+    # A group of ASLine objects with logic to correctly turn them into a MARC::Record object
+    # @see ASLine
+    
+    class ASLineGroup
+      
+      include Log
+      
+      # @!attribute aslines
+      #   @return [Array<MARC::Field>] Internal list of MARC field object
+      attr_accessor :aslines
+      
+      # @!attribute [r] leader
+      #   @return [String] The leader string, pulled from whatever was passed in with a LDR tag
+      attr_reader   :leader
+      
+      
+      
+      def initialize
+        @aslines = []
+        @leader = nil
+      end
+      
+      # Number of aslines already added
+      # @return Integer
+      def size
+        aslines.size
+      end
+      
+      # Is this group empty?
+      def empty?
+        aslines.empty?
+      end
+      
+      
+      # Add an ASLine object, turning it into the appropriate type of field as we go
+      # An ASLine object with type :invalid_id will be treated as a string and appended to
+      # the previous field (to deal with not-uncommon spurious newlines in data fields)
+      # @return [Undefined] side effect only
+      # @raise MARC::AlephSequential::Error when there's an invalid ID _and_ there's no previous
+      # field to concatentate it to.
+      
+      def add(asline)
+        case asline.type
+        when :leader
+          if leader
+            log.warn("#{asline.line_number} #{asline.id} Set leader more than once; last one wins")
+          end
+          @leader = asline.value
+        when :invalid_id
+          lastfield = @aslines.pop
+          unless lastfield
+            raise MARC::AlephSequential::Error.new('unknown', asline.line_number),  
+                  "#{asline.line_number} has invalid id and no preivous line to concat it to (file starts bad?)"
+                  nil
+          end
+          log.info "#{asline.line_number} #{lastfield.id} / #{lastfield.tag} Concatenating line #{asline.line_number} to previous line"
+          @aslines.push ASLine.new(lastfield.rawstr +  asline.rawstr, lastfield.line_number)
+        else
+          @aslines.push asline
+        end
+      end
+      
+      
+      # Add an asline as a raw string
+      def add_string(asline_string, line_number)
+        self.add(ASLine.new(asline_string, line_number))
+      end
+      
+      # Turn this object into a MARC::Record
+      # @return [MARC::Record]
+      # @raise MARC::AlephSequential::Error if this object is empty
+      # @raise MARC::AlephSequential::Error if there's no leader
+      def as_record
+        if empty?
+          raise MARC::AlephSequential::Error.new('unknown', 'unknown'), "Can't turn an empty group into a record", nil
+        end
+        
+        unless leader
+          raise MARC::AlephSequential::Error.new(@aslines[0].id, @aslines[0].line_number),
+                "Record #{@aslines[0].id} (near line #{ @aslines[0].line_number}) has no leader; can't turn into a record",
+                nil
+        end
+        r = MARC::Record.new
+        r.leader = leader
+        aslines.map {|f| r << f.to_field}
+        return r
+      end
+      
+      alias_method :to_record, :as_record
+    end
+  end
+end
+          

data/lib/marc_alephsequential/asline_reader.rb

@@ -0,0 +1,16 @@
+require_relative 'asline'
+require_relative 'buffered_linereader'
+
+module MARC
+  module AlephSequential
+
+    class ASLineReader < BufferedLineReader
+      def process_raw(raw, line_number)
+        super
+        ASLine.new(raw, line_number)
+      end
+      
+    end
+  end
+end
+      

data/lib/marc_alephsequential/buffered_linereader.rb

@@ -0,0 +1,100 @@
+require 'zlib'
+
+module MARC
+  module AlephSequential
+
+    # AlephSequential is a line-oriented format, with the first field of each line 
+    # indicating the record number. Rather than try to screw around with keeping track of
+    # the last line read, checking to see if we have one, blah blah blah, I'm going to use
+    # a buffered line reader class so I can #peek at the next line to know if its id 
+    # is different than the current record. 
+    
+    class BufferedLineReader
+      
+      include Enumerable
+      
+      attr_accessor :buffer_size
+      attr_reader :underlying_line_number
+      
+      def initialize(filename_or_io)
+        
+        @passed_in = filename_or_io
+        
+        @underlying_line_number = 0
+        @buffer_size = 10
+        @buffer = []
+        
+        if filename_or_io.is_a? String
+          @handle = File.open(filename_or_io, 'r:utf-8')
+          if filename_or_io =~ /\.gz$/
+            @handle = Zlib::GzipReader.new(@handle)
+          end
+        elsif filename_or_io.respond_to?("read", 5)
+          @handle = filename_or_io
+        else
+          raise ArgumentError.new("BufferedLineReader needs an IO object or filename, got #{filename_or_io} (#{filename_or_io.inspect})")
+        end
+        
+        @iter = @handle.enum_for(:each_line)
+        @finished = false
+        # Fill up the buffer
+        self.fillbuffer
+      end
+      
+      def has_next?
+        return !(@finished && @buffer.size == 0)
+      end
+      
+      def fillbuffer(buffer_size = @buffer_size)
+        begin
+          buffer_size.times do
+            raw = @iter.next
+            @underlying_line_number += 1
+            @buffer.push process_raw(raw, @underlying_line_number)
+          end
+        rescue StopIteration
+          @finished = true
+        end
+      end
+      
+      # Empty version here; can override for processing lines on the fly
+      def process_raw(raw, line_number)
+        raw
+      end
+            
+      def next
+        raise StopIteration, "End of #{@passed_in}", nil if @buffer.size == 0
+        rv = @buffer.shift
+        fillbuffer if @buffer.size == 0
+        rv
+      end
+        
+      
+      def peek
+        fillbuffer unless @buffer.size > 0
+        @buffer[0]
+      end
+            
+      def each
+        begin
+          while true
+            yield self.next
+          end
+        rescue StopIteration
+        end
+      end
+      
+      alias_method :each_line, :each
+    end
+  end
+end
+        
+        
+        
+      
+      
+      
+      
+      
+      
+      

data/lib/marc_alephsequential/error.rb

@@ -0,0 +1,13 @@
+module MARC
+  module AlephSequential
+    
+    class Error < RuntimeError
+      attr_accessor :record_id, :line_number
+  
+      def initialize(record_id, line_number)
+        @record_id = record_id
+        @line_number = line_number
+      end
+    end
+  end
+end

data/lib/marc_alephsequential/log.rb

@@ -0,0 +1,20 @@
+module MARC
+  module AlephSequential
+    
+    module Log
+    
+      def self.log
+        @log
+      end
+    
+      def self.log=(log)
+        @log = log
+      end
+    
+      def log
+        Log.log ||= Yell.new STDERR, :level => [ :warn, :error], :name=>'MAS'
+        Log.log
+      end
+    end
+  end
+end