marcspec 0.2.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 BillDueber
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,59 @@
+= marcspec
+
+The MARCSpec contains classes designed to make it (relatively) easy to specify
+a data field (my use case specifically is solr) in terms of sets of MARC fields and subfields.
+
+== A simple breakdown of the hierarchy
+
+This is all based on how the excellent [Solrmarc](http://code.google.com/p/solrmarc/)
+deals with configuration. MARCSpec supports a subset of Solrmarc's configuration
+options. 
+
+* A {MARCSpec::SpecSet} consists of a (possibly empty) set of named {MARCSpec::Map}
+  and a list of {MARCSpec::SolrFieldSpec} and/or {MARCSpec::CustomSolrSpec}. 
+* A {MARCSpec::SolrFieldSpec} consists of a field name, a list of MARC field specs (see below),
+  an optional {MARCSpec::Map} for translating raw values to something else, and
+  a bunch of optional specials (e.g., a notation to only use the first value,
+  a default value if no appropriate data is in the MARC record, a default value
+  for when marc data is found, but nothing is in the map, etc.)
+* A {MARCSpec::CustomSolrSpec} occupies the same niche as a {MARCSpec::SolrFieldSpec}, but
+  allows you to use a custom routine instead of the pre-packaged MARC specification syntax.
+* A MARC Field Spec is one of the following
+  * A {MARCSpec::LeaderSpec} for dealing with the leader
+  * A {MARCSpec::ControlFieldSpec}, consisting of a tag (as a string, not a number) and
+    an optional zero-based index (e.g., "001[3]") or range (e.g., "001\[11..13\]")
+  * A {MARCSpec::VariableFieldSpec}, consisting of a tag, a couple indicator patterns
+    (currently ignored, but stay tuned), an optional list of subfields (default
+    is all), and an optional string used to join the subfields (default is
+    a single space)
+* A *map* is one of:
+  * A {MARCSpec::KVMap}, which is just a Ruby hash. It will match at most one k-v pair, although
+    the "value" might actually be either a scalar or an array of scalars
+  * A {MARCSpec::MultiValueMap}, which is an array of key/value duples. A passed
+    potential map is compared (via ===) with every key, returning all
+    the associated values. Again, a single "key" can be associated with an array of
+    return values; the whole thing is flattened out before returning
+
+Obviously, better descriptions and full documentation is available in each 
+individual class.
+
+== Better examples in marc2solr
+
+Better documented samples are available as part of the marc2solr project
+at http://github.com/billdueber/marc2solr -- look in the simple_sample area.
+
+
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 BillDueber. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "marcspec"
+    gem.summary = %Q{Extract data from MARC records and send to Solr}
+    gem.description = %Q{Relies on marc4j4r, based on work in solrmarc}
+    gem.email = "bill@dueber.com"
+    gem.homepage = "http://github.com/billdueber/marcspec"
+    gem.authors = ["BillDueber"]
+    gem.add_development_dependency "bacon", ">= 0"
+    gem.add_development_dependency "yard", ">= 0"
+
+    gem.add_dependency 'marc4j4r', '>=0.9.0'
+    gem.add_dependency 'jruby_streaming_update_solr_server', '>=0.2.0'
+    
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |spec|
+    spec.libs << 'spec'
+    spec.pattern = 'spec/**/*_spec.rb'
+    spec.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.1

data/lib/marcspec/customspec.rb

@@ -0,0 +1,97 @@
+require 'marcspec/map'
+require 'marcspec/solrfieldspec'
+
+
+module MARCSpec
+
+  # A CustomSolrSpec is a SolrFieldSpec that derives all its values from a custom function. The custom function
+  # must me a module function that takes a record and an array of other arguments and returns a 
+  # (possibly empty) list of resulting values.
+  #
+  # @example
+  #  module MARC2Solr
+  #   module MyCustomStuff
+  #     def self.uppercaseTitle r, args=[]
+  #       vals = []
+  #       vals.push r['245'].value.upcase
+  #       return vals
+  #     end
+  #   end
+  # end
+  #
+  # css = MARCSpec::CustomSolrSpec.new(:module => MARC2Solr::MyCustomStuff,
+  #                                    :methodSymbol => :uppercaseTitle,
+  #                                    :map => ss.map('mapname')
+  #                                   )
+  # ss.add_spec(css)
+  #
+  # 
+  
+
+  class CustomSolrSpec < SolrFieldSpec
+    
+    attr_accessor :module, :methodSymbol, :methodArgs
+    def initialize(opts)
+      @solrField  = opts[:solrField]
+      @module = opts[:module]
+      @methodSymbol = opts[:methodSymbol]
+
+      unless @solrField and @module and @methodSymbol
+        raise ArgumentError, "Custom solr spec must have a field name in :solrField, module in :module, and the method name as a symbol in :methodSymbol"
+      end
+      
+      @methodArgs = opts[:methodArgs] || []
+      
+      @first = opts[:firstOnly] || false      
+      @default = opts[:default] || nil
+      @map = opts[:map] || nil
+      @noMapKeyDefault = opts[:noMapKeyDefault] || nil
+      
+    end
+    
+    def raw_marc_values r
+      return @module.send(@methodSymbol, r, *@methodArgs)
+    end
+    
+    def self.fromHash h
+      return self.new(h)
+    end
+  
+    
+    def asPPString
+      s = StringIO.new
+      s.print "{\n :solrField=> "
+      PP.singleline_pp(@solrField, s)
+      s.print(",\n ")
+      s.print ":firstOnly => true,\n " if @first
+      if @default
+        s.print(":default => ")
+        PP.singleline_pp(@default, s)
+        s.print(",\n ")
+      end
+      if @map
+        s.print(":mapname => ")
+        PP.singleline_pp(@map.mapname, s)
+        s.print(",\n ")
+      end
+      if @noMapKeyDefault
+        s.print(":noMapKeyDefault => ")
+        PP.singleline_pp(@noMapKeyDefault, s)
+        s.print(",\n ")
+      end
+      
+      s.print(":module => ")
+      PP.singleline_pp(@module, s)
+      s.print(",\n :methodSymbol => ")
+      PP.singleline_pp(@methodSymbol, s)
+      if @methodArgs
+        s.print(",\n :methodArgs => ")
+        PP.singleline_pp(@methodArgs, s)
+      end
+      s.print "\n}"
+      return  s.string
+    end
+    
+  end
+end
+    

data/lib/marcspec/kvmap.rb

@@ -0,0 +1,79 @@
+require 'marcspec/map'
+require 'pp'
+
+
+
+
+module MARCSpec
+
+  # A KVMap is, when push comes to shove, just a hash with a name, and the 
+  # option of adding a default value for each lookup.
+  
+  class KVMap < Map
+    
+    # Basic lookup which takes a lookup key and an optional default value,
+    # which will be returned iff the map doesn't have
+    # the passed key
+    #
+    # @example
+    #   kvmap = MARCSpec::KVMap.new("sample_map", {1=>'one'})
+    #   kvmap[1]  #=> 'one'
+    #   kvmap[2]  #=> nil
+    #   kvmap[2, 'Not Found'] #=> 'Not Found'
+    #
+    # @param [Object] key The key to look up
+    # @param [Object] default The value to return if the lookup fails
+    # @return [Object] The value associated with the passed key, or the
+    # default value
+
+    def [] key, default=nil
+      if @map.has_key? key
+        @map[key]
+      else
+        if default == :passthrough
+          return key
+        else
+          return default
+        end
+      end
+    end    
+    
+    def []= key, value
+      @map[key] = value
+    end
+    
+    alias_method :add, :[]=
+    
+    def asPPString
+      s = StringIO.new
+      s.print "{\n :maptype=>:kv,\n :mapname=>"
+      PP.singleline_pp(@mapname, s)
+      s.print ",\n :map => "
+      PP.pp(@map, s)
+      s.puts "\n}"
+      return s.string
+    end
+    
+        
+    def self.from_solrmarc_file filename
+      mapname = File.basename(filename).sub(/\..+?$/, '')
+      map = {}
+      File.open(filename) do |smf|
+        smf.each_line do |l|
+          l.chomp!
+          next unless l =~ /\S/
+          l.strip!
+          next if l =~ /^#/
+          unless l =~ /^(.+?)\s*=\s*(.+)$/
+            $LOG.warn "KVMap import skipping weird line in #{filename}\n  #{l}"
+            next
+          end
+          map[$1] = $2
+        end
+      end
+      return self.new(mapname, map)
+    end
+    
+    
+  end
+end

data/lib/marcspec/map.rb

@@ -0,0 +1,67 @@
+module MARCSpec
+  
+  # A Map is just a named lookup table. The access
+  # (via []) takes, in adition to a key, an optional
+  # default value to return 
+  
+  class Map
+    attr_accessor :mapname, :map
+    
+    # Create a new map. The passed map is either
+    # a standard hash or a list of duples
+    #
+    # @param
+    def initialize(mapname, map)
+      @mapname = mapname
+      @map = map
+    end
+
+    def self.fromFile filename
+      begin
+        str = File.open(filename).read
+      rescue Exception => e
+        $LOG.error "Problem opening #{filename}: #{e.message}"
+        raise e
+      end
+      
+      begin 
+        rawmap = eval(str)
+      rescue Exception => e
+        $LOG.error "Problem evaluating (with 'eval') file #{filename}: #{e.message}"
+        raise e
+      end
+      
+      case rawmap[:maptype]
+      when :kv
+        return KVMap.new(rawmap[:mapname], rawmap[:map])
+      when :multi
+        return MultiValueMap.new(rawmap[:mapname], rawmap[:map])
+      else
+        $LOG.error "Map file #{filename} doesn't seem to be either a KV map or a MuliValueMap according to :maptype (#{rawmap[:maptype]})"
+        raise ArgumentError, "File #{filename} doesn't evaluate to a valid map"
+      end
+      
+    end
+      
+
+    def == other
+      return ((other.mapname == self.mapname) and (other.map = self.map))
+    end
+    
+    def pretty_print pp
+      pp.pp eval(self.asPPString)
+    end
+    
+    def self.fromHash rawmap
+      return self.new(rawmap[:mapname], rawmap[:map])
+    end
+    
+    # Take the output of pretty_print and eval it to get rawmap; pass it
+    # tp fromHash to get the map object
+    def self.fromPPString str
+      rawmap = eval(str)
+      return self.fromHash rawmap
+    end
+    
+  end
+end    

data/lib/marcspec/marcfieldspec.rb

@@ -0,0 +1,205 @@
+require 'marc4j4r'
+require 'set'
+require 'pp'
+module MARCSpec
+
+  # A ControlFieldSpec takes a control tag (generally 001..009) and an optional zero-based range
+  # When called with marc_values(record), it returns either the complete value of all
+  # occurances of the field in question (in the order they appear in the record), or 
+  # the zero-based substrings based on the passed range. 
+  #
+  # @example Get the whole 001
+  # cfs = MARCSpec::ControlTagSpec.new('001')
+  # 
+  # @example Get the first three characters of the 008
+  # cfs = MARCSpec::ControlTagSpec.new('001', 0..2)
+  #
+  # Note that the use of the zero-based range in this manner conforms to the way MARC 
+  # substrings are specified.
+  
+  class ControlFieldSpec
+    attr_accessor :tag, :range
+    
+    def initialize (tag, range=nil)
+      unless MARC4J4R::ControlField.control_tag? tag
+        raise ArgumentError "Tag must be a control tag"
+      end
+      @tag = tag
+      self.range = range
+    end
+    
+    def == other
+      return ((self.tag == other.tag) and
+              (self.range = other.range))
+    end
+    
+    
+    # Always force a real range, since in Ruby 1.9 a string subscript with a single fixnum
+    # will return the character code of that character (e.g., "Bill"[0] => 66, wherease 
+    # "Bill"[0..0] gives the expected 'B'
+    #
+    # @param [nil, Fixnum, Range] range A zero-based substring range or character position
+    # @return [MARCSpec::ControlFieldSpec] self
+    
+    def range= range
+      if range.nil?
+        @range = nil
+        return self
+      end
+      if range.is_a? Fixnum
+        if range < 0
+          raise ArgumentError, "Range must be nil, an integer offset (1-based), or a Range, not #{range}"
+        end
+      
+        @range = range..range
+      
+      elsif range.is_a? Range
+        @range = range
+      else
+        raise ArgumentError, "Range must be nil, an integer offset (1-based), or a Range, not #{range.inspect}"
+      end
+      return self
+    end
+    
+    
+    def marc_values r
+      vals = r.find_by_tag(@tag).map {|f| f.value}
+      if @range
+        return vals.map {|v| v[@range]}
+      else
+        return vals
+      end
+    end
+    
+    def pretty_print pp
+      pp.pp eval(self.asPPString)
+    end
+    
+    def asPPString
+      s = StringIO.new
+      if @range
+        PP.pp([@tag, @range], s)
+      else
+        PP.pp([@tag], s)
+      end
+      return s.string
+    end
+    
+    def self.fromPPString str
+      a = eval(str)
+      return self.new(*a)
+    end
+    
+  end
+  
+  
+  # A LeaderSpec deals only with the leader. It's basically the same as a controlfield spec, but
+  # using the string 'LDR' to identify itself
+  
+  class LeaderSpec < ControlFieldSpec
+    
+    # Built to be syntax-compatible with ControlFieldSpec, the tag must always
+    # be 'LDR' (case matters)
+    #
+    # @param ['LDR'] tag The 'tag'; in this case, always 'LDR'
+    # @param [Fixnum, Range<Fixnum>] range substring specification (either one character or a range) to return
+    # instead of the whole leader.
+    
+    def initialize (tag, range=nil)
+      unless tag == 'LDR'
+        raise ArgumentError "Tag must be 'LDR'"
+      end
+      @tag = 'LDR'
+      self.range = range
+    end
+    
+    # Return the appropriate value (either the leader or a subset of it) from the
+    # given record
+    #
+    # @param [MARC4J4R::Record] r A MARC4J4R Record
+    # @return [String] the leader or substring of the leader
+    def marc_values r
+      if @range
+        return r.leader[@range]
+      else
+        return r.leader
+      end
+    end
+  end    
+    
+  
+  # A VariableFieldSpec has a tag (three chars) and a set of codes. Its #marc_values(r) method will return
+  # all the values for the subfields for the given codes joined by the optional joiner (space by default)
+  #
+  # The subfield values are presented in the order they appear in the document, *not* the order the subfield
+  # codes are specified
+  #
+  # @example Get the $a from the 245s
+  # vfs = MARCSpec::VariableFieldSpec.new('245', 'a')
+  #
+  # vfs = MARCSpec::VariableFieldSpec.new('245', 'ab')
+  
+  
+  class VariableFieldSpec
+    
+    attr_accessor :tag, :codes, :joiner
+    
+    def initialize tag, codes=nil, joiner=' '
+      @tag = tag
+      @joiner = joiner || ' '
+      self.codes = codes
+    end
+    
+    def == other
+      return ((self.tag == other.tag) and
+              (self.codes = other.codes) and
+              (self.joiner = other.joiner))
+    end
+    
+    def codes= c
+      if c.nil?
+        @codes = nil
+        return nil
+      end
+      
+      if( c.is_a? Array) or (c.is_a? Set) or (c.is_a? Range)
+        @codes = c.to_a
+      else
+        @codes = c.split(//)
+      end
+
+      return @codes
+    end
+    
+    def marc_values r
+      fields = r.find_by_tag(@tag)
+      vals = []
+      fields.each do |f|
+        subvals = f.sub_values(@codes)
+        vals << subvals.join(@joiner) if subvals.size > 0
+      end
+      return vals
+    end
+
+    def pretty_print pp
+      pp.pp eval(self.asPPString)
+    end
+    
+    def asPPString
+      s = StringIO.new
+      if @joiner and @joiner != ' '
+        PP.pp([@tag, '*', '*', @codes.join(''), @joiner], s)
+      else
+        PP.pp([@tag, '*', '*', @codes.join('')], s)
+      end
+      return s.string
+    end
+
+   def self.fromPPString str
+     a = eval(str)
+     return self.new(a[0], a[3], a[4])
+   end
+
+  end
+  
+end   

data/lib/marcspec/multivaluemap.rb

@@ -0,0 +1,62 @@
+require 'marcspec/map'
+require 'pp'
+
+
+module MARCSpec
+
+  # A MultiValueMap (in this conectex) is an array of duples of the form
+  # [thingToMatch, 'Value'] (or [thingToMatch, [array,of,values]])
+  # along with an associated name.
+  #
+  # Accessing via [] will give you an array of non-nil values that match (via ===)
+  # the corresponding keys.
+  #
+  # Keys can be either strings or regular expressions (e.g., /^Bil/). 
+  #
+  # Again, note that if several keys are === to the passed argument, all the values will be returned. 
+  
+  class MultiValueMap   < Map
+    
+    attr_accessor :mapname,:map
+    
+
+    def [] key, default=nil
+      rv =  @map.map {|pv| pv[0] === key ? pv[1] : nil}
+      rv.flatten!
+      rv.compact!
+      rv.uniq!
+      if rv.size > 0
+        return rv
+      else
+        return [default]
+      end
+    end
+    
+    def self.from_solrmarc_file filename
+      mapname = File.basename(filename).sub(/\..+?$/, '')
+      kvlist = []
+      File.open(filename) do |f|
+        f.each_line do |line|
+          match = /^pattern.*?=\s*(.*?)\s*=>\s*(.*?)\s*$/.match(line)
+          unless match
+            $LOG.warn "MultiValueMap import skipping weird line in #{filename}\n  #{l}"
+            next
+          end
+          kvlist << [Regexp.new(match[1]), match[2]]
+        end
+      end
+      return self.new(mapname, kvlist)
+    end
+        
+    def asPPString
+      s = StringIO.new
+      s.print "{\n :maptype=>:multi,\n :mapname=>"
+      PP.singleline_pp(@mapname, s)
+      s.print ",\n :map => "
+      PP.pp(@map, s)
+      s.puts "\n}"
+      return s.string
+    end
+  end
+end
+