marcspec 0.2.1

data/lib/marcspec/solrfieldspec.rb

@@ -0,0 +1,123 @@
+require 'stringio'
+module MARCSpec
+  class SolrFieldSpec
+    attr_accessor :solrField, :first, :map, :noMapKeyDefault, :marcfieldspecs, :default
+
+    def initialize(opts)
+      @solrField  = opts[:solrField]
+      @first = opts[:firstOnly] || false      
+      @default = opts[:default] || nil
+      @map = opts[:map] || nil
+      @noMapKeyDefault = opts[:noMapKeyDefault] || nil
+      @marcfieldspecs = []
+    end
+
+    def << tagspec
+      @marcfieldspecs << tagspec
+    end
+
+
+    def raw_marc_values r
+      vals = []
+      @marcfieldspecs.each do |ts|
+        vals.concat ts.marc_values(r)
+      end
+      return vals
+    end
+      
+    def marc_values r
+      vals = raw_marc_values r
+      unless vals.is_a? Array
+        vals = [vals]
+      end
+      
+      if @first
+        vals = [vals.compact.first].compact
+      end
+
+      # If we got nothing, just return either nothing or the defualt,
+      # if there is one. Don't screw around with mapping.
+      if vals.size == 0
+        if @default.nil? # unless there's a default value, just return nothing
+          return []
+        else
+          return [@default]
+        end
+      end
+      
+      # If we've got a map, map it.
+
+      if (@map)
+        vals.map! {|v| @map[v, @noMapKeyDefault]}
+      end
+      
+      # Flatten it all out
+      
+      vals.flatten!
+      vals.uniq!
+      vals.compact!
+      return vals
+    end
+    
+    
+    def == other
+      return ((other.solrField == self.solrField) and
+             (other.first == self.first) and
+             (other.map == self.map) and
+             (other.noMapKeyDefault == self.noMapKeyDefault) and
+             (other.marcfieldspecs == self.marcfieldspecs))
+    end
+    
+    def self.fromPPString str
+      return self.fromHash eval(str)
+    end
+    
+    def self.fromHash h
+      sfs = self.new(h)
+      h[:specs].each do |s|
+        if s.size < 3 
+          sfs << MARCSpec::ControlFieldSpec.new(*s)
+        else
+          sfs << MARCSpec::VariableFieldSpec.new(s[0], s[3], s[4])
+        end
+      end
+      return sfs
+    end
+    
+    def pretty_print pp
+      pp.pp eval(self.asPPString)
+    end
+    
+    def asPPString
+      s = StringIO.new
+      s.print "{\n :solrField=> "
+      PP.singleline_pp(@solrField, s)
+      s.print(",\n ")
+      s.print ":firstOnly => true,\n " if @first
+      if @default
+        s.print(":default => ")
+        PP.singleline_pp(@default, s)
+        s.print(",\n ")
+      end
+      if @map
+        s.print(":mapname => ")
+        PP.singleline_pp(@map.mapname, s)
+        s.print(",\n ")
+      end
+      if @noMapKeyDefault
+        s.print(":noMapKeyDefault => ")
+        PP.singleline_pp(@noMapKeyDefault, s)
+        s.print(",\n ")
+      end
+      s.print(":specs => [\n")
+      @marcfieldspecs.each do |ts|
+        s.print '  '
+        PP.singleline_pp(ts, s)
+        s.print(",\n")
+      end
+      s.print " ]\n}"
+      return  s.string
+    end
+
+  end
+end    

data/lib/marcspec/specset.rb

@@ -0,0 +1,58 @@
+require 'jruby_streaming_update_solr_server'
+module MARCSpec
+  class SpecSet
+    attr_accessor :tmaps, :solrfieldspecs
+    
+    def initialize
+      @tmaps = {}
+      @solrfieldspecs = []
+    end
+    
+    def map name
+      return self.tmaps[name]
+    end
+    
+    def loadMapsFromDir dir
+      unless File.exist? dir
+        raise ArgumentError, "Cannot load maps from #{dir}: does not exist"
+      end
+      Dir.glob("#{dir}/*.rb").each do |tmapfile|
+        self.add_map(MARCSpec::Map.fromFile(tmapfile))
+      end
+    end
+    
+    def add_map map
+      self.tmaps[map.mapname] = map
+    end
+    
+    def add_spec solrfieldspec
+      self.solrfieldspecs << solrfieldspec
+    end
+    
+    alias_method :<<, :add_spec
+
+    def each
+      @solrfieldspecs.each do |fs|
+        yield fs
+      end
+    end
+    
+    def fill_hashlike_from_marc r, hashlike
+      @solrfieldspecs.each do |sfs|
+        hashlike[sfs.solrField] = sfs.marc_values(r)
+      end
+    end
+
+    def doc_from_marc r
+      doc = SolrInputDocument.new
+      fill_hashlike_from_marc r, doc
+      return doc
+    end      
+    
+    def hash_from_marc r
+      h = {}
+      fill_hashlike_from_marc r, h
+      return h
+    end
+  end
+end

data/lib/marcspec.rb

@@ -0,0 +1,11 @@
+require 'logger'
+
+$LOG = Logger.new(STDOUT)
+$LOG.level = Logger::WARN
+
+require "marcspec/customspec"
+require "marcspec/solrfieldspec"
+require "marcspec/kvmap"
+require "marcspec/multivaluemap"
+require "marcspec/specset"
+require "marcspec/marcfieldspec"

data/lib/orig.rb

@@ -0,0 +1,288 @@
+require 'set'
+require 'pp'
+require 'logger'
+
+$LOG ||= Logger.new(STDERR)
+
+module MARCSpec
+
+  class MapSpec
+    attr_accessor :map, :type, :default
+    
+    def initialize(type, map, default=nil)
+      @type = type
+      @default = default
+      @map = map
+    end
+    
+    def [] key
+      if (@type == :kv)
+        if @map.has_key? key
+          return @map[key]
+        else
+          return @default
+        end
+      end
+      
+      # For a pattern, we find all that match. 
+      
+      if (@type == :pattern)
+        rv = []
+        @map.each do |pv|
+          pat = pv[0]
+          val = pv[1]
+#          puts "Trying pattern #{pat} against #{key}"
+          if pat.match(key)
+            rv << val
+#            puts "Matched: adding #{val}"
+          end
+        end
+        rv.uniq!
+        if rv.size > 0
+          return  rv
+        else 
+          return @default
+        end
+      end
+    end
+  end
+  
+  class CustomSpec
+    def initialize(proc, args)
+      @proc = proc
+      @args = args
+    end
+    
+    def marc_values_hash fieldnames, r
+      a = @proc(r, args)
+      rv = {}
+      fieldnames.each_with_index do |fn, i|
+        rv[fn] = a[i]
+      end
+      return rv
+    end
+  end
+      
+    
+  class TagSpec
+    attr_accessor :tag, :codes, :joiner, :parent, :ind1, :ind2, :range, :is_control
+    
+    def initialize(tag, codes=nil)
+      @codes = Set.new
+      @tag = tag
+      @joiner = ' '
+      @substr = nil
+      tagint = tag.to_i
+      @is_control = (tagint != 0 and tagint < 10)
+      if (codes)
+        self.codes = codes
+      end
+    end
+    
+    def range= newrange
+      if newrange =~ /\s*(\d+)-(\d+)/
+        start = $1.to_i 
+        last =  $2.to_i
+        @range = start..last
+      else
+        se = newrange.to_i
+        @range = se..se
+      end
+    end
+    
+    def codes= newcodes
+      if newcodes.is_a? Range
+        @codes = newcodes.to_a
+      elsif newcodes !~ /\S/
+        @codes = nil
+      # Otherwise, just split into individual characters
+      else
+        @codes = newcodes.split(//)
+      end
+    end
+    
+    def marc_values r
+      if @is_control
+        vals = r.find_by_tag(@tag).map {|f| f.value}
+#        puts "Start with #{vals.join(', ')}"
+        if @range
+          vals.map! {|v| v[@range]}
+        end
+#        puts "End with #{vals.join(', ')}"
+        
+      else 
+        fields = r.find_by_tag(@tag)
+        vals = []
+        fields.each do |f|
+          subvals = f.sub_values(@codes)
+          vals << subvals.join(@joiner) if subvals.size > 0
+        end
+      end
+      # puts vals.join(', ')
+      return vals
+    end
+    
+  end
+  
+  class FieldSpec
+    attr_accessor :field, :first, :map, :tagspecs
+    
+    def initialize(opts)
+      @field  = opts[:field]
+      @first = opts[:first] || false
+      @map = opts[:map] || nil
+      @tagspecs = []
+    end
+    
+    def << tagspec
+      tagspec.parent = self
+      @tagspecs << tagspec
+    end
+    
+    def marc_values r
+      vals = []
+      # puts "Tagspecs has #{@tagspecs.size} items"
+      @tagspecs.each do |ts|
+        vals.concat ts.marc_values(r)
+        # puts vals.join(', ')
+        break if @first and vals.size > 0
+      end
+      
+      if (@map)
+        vals.map! {|v| @map[v]}
+        # vals.each do |v|
+        #   puts "Map: #{v} => #{@map[v].to_s}"
+        # end
+      end
+      vals.flatten!
+      vals.uniq!
+      vals.compact!
+      return vals
+    end
+    
+  end
+  
+  
+  class SpecSet
+    attr_accessor :tmaps, :fieldspecs
+    def initialize(*args)
+      tmapdir = args.pop!
+      unless File.directory? tmapdir
+        $LOG.error "Directory #{tmapdir} not found"
+        raise LoadError, "Directory #{tmapdir} not found"
+      end
+              
+      @tmaps = {}
+      Dir.glob(tmapdir + '/*.rb') do |fn|
+        basename = File.basename(fn).sub(/\.rb$/, '')
+        $LOG.info "Loading translation map #{basename}"
+        
+        begin
+          rawmap = eval(File.open(fn).read)
+          @tmaps[basename] = MapSpec.new(rawmap[:type], rawmap[:map], rawmap[:default])
+        rescue SyntaxError
+          $LOG.error "Error processing translation map file #{fn}: #{$!}"
+          raise SyntaxError, $!
+        end
+        
+      end
+      
+      @fieldspecs = []
+      
+    # Get the index files
+      args.each do |indexfile|
+        begin
+          unless File.exists? indexfile
+            $LOG.error "File #{indexfile} does not exist"
+            raise LoadError, "File #{indexfile} does not exist"
+          end
+          $LOG.info "Loading index file #{indexfile}"
+          rawindex = eval(File.open(indexfile).read)
+          rawindex.each do |entry|
+            fs = FieldSpec.new(:field => entry[:solrField], :first=>entry[:firstOnly])
+            mapname = entry[:map]
+            if mapname
+              if @tmaps.has_key? mapname
+                fs.map = @tmaps[mapname]
+              else 
+                $LOG.error "Can't find map #{mapname}"
+              end
+            end
+            entry[:specs].each do |entryts|
+            
+              # A one- or two-element entry is a control field
+              # A three element entry (tag, ind1, ind2) is all subs of a field (need to implement)
+              # A four element field is tag, ind1, ind2, subs
+              # A five element field is tag, ind1, ind2, subs, joiner
+              
+            
+              tag = entryts[0]
+            
+              # Is tag the symbol :custom? Then make it a custom item
+            
+              if tag == :custom
+                ts = CustomSpec.new(entryts[1], entryts[2..-1])
+                fs << ts
+                next
+              end
+            
+              # If it's not custom, the solrField better be a scale
+              if entry[:solrField].is_a? Array
+                # log an error and bail out
+              end
+            
+              # Otherwise, it's a tag spec
+              if tag.is_a? Fixnum
+                tag = '%03d' % tag
+              end
+            
+            
+              ts = TagSpec.new(tag)
+              if entryts.size < 3
+                ts.is_control = true
+                ts.range = entryts[1] if entryts[1]
+              else
+                ts.ind1 = entryts[1]
+                ts.ind2 = entryts[2]            
+                ts.codes = entryts[3]
+                ts.joiner = entryts[4] if entryts[4]
+              end
+              fs << ts
+            end
+            self << fs
+          end
+        rescue SyntaxError
+          $LOG.error "Error processing index file #{indexfile}: #{$!}"
+          raise SyntaxError
+        end      
+      end
+    end
+    
+    def each
+      @fieldspecs.each do |fs|
+        yield fs
+      end
+    end
+    
+    def << fieldspec
+      @fieldspecs << fieldspec
+    end
+    
+    def doc_from_marc r
+      doc = SolrInputDocument.new
+      @fieldspecs.each do |fs|
+        doc[fs.field] = fs.marc_values(r)
+      end
+      return doc
+    end      
+    
+    def hash_from_marc r
+      h = {}
+      @fieldspecs.each do |fs|
+       h[fs.field] = fs.marc_values(r)
+      end
+      return h
+    end
+    
+  end
+end  

data/spec/data/batch.dat

@@ -0,0 +1 @@
+00755cam  22002414a 4500001001300000003000600013005001700019008004100036010001700077020004300094040001800137042000800155050002600163082001700189100003100206245005400237260004200291300007200333500003300405650003700438630002500475630001300500fol05731351 IMchF20000613133448.0000107s2000    nyua          001 0 eng    a   00020737   a0471383147 (paper/cd-rom : alk. paper)  aDLCcDLCdDLC  apcc00aQA76.73.P22bM33 200000a005.13/32211 aMartinsson, Tobias,d1976-10aActivePerl with ASP and ADO /cTobias Martinsson.  aNew York :bJohn Wiley & Sons,c2000.  axxi, 289 p. :bill. ;c23 cm. +e1 computer  laser disc (4 3/4 in.)  a"Wiley Computer Publishing." 0aPerl (Computer program language)00aActive server pages.00aActiveX.00647pam  2200241 a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001800109042000800127050002600135082001500161100002600176245006700202260003800269263000900307300001100316650003700327650002500364700001600389fol05754809 IMchF20000601115601.0000203s2000    mau           001 0 eng    a   00022023   a1565926994  aDLCcDLCdDLC  apcc00aQA76.73.P22bD47 200000a005.742211 aDescartes, Alligator.10aProgramming the Perl DBI /cAlligator Descartes and Tim Bunce.  aCmabridge, MA :bO'Reilly,c2000.  a1111  ap. cm. 0aPerl (Computer program language) 0aDatabase management.1 aBunce, Tim.00605cam  22002054a 4500001001300000003000600013005001700019008004100036010001700077040001800094042000800112050002700120082001700147100002100164245005500185260004500240300002600285504005100311650003700362fol05843555 IMchF20000525142739.0000318s1999    cau      b    001 0 eng    a   00501349   aDLCcDLCdDLC  apcc00aQA76.73.P22bB763 199900a005.13/32211 aBrown, Martin C.10aPerl :bprogrammer's reference /cMartin C. Brown.  aBerkeley :bOsborne/McGraw-Hill,cc1999.  axix, 380 p. ;c22 cm.  aIncludes bibliographical references and index. 0aPerl (Computer program language)00579cam  22002054a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001800109042000800127050002700135082001700162100002100179245005500200260004500255300003600300650003700336fol05843579 IMchF20000525142716.0000318s1999    caua          001 0 eng    a   00502116   a0072120002  aDLCcDLCdDLC  apcc00aQA76.73.P22bB762 199900a005.13/32211 aBrown, Martin C.10aPerl :bthe complete reference /cMartin C. Brown.  aBerkeley :bOsborne/McGraw-Hill,cc1999.  axxxv, 1179 p. :bill. ;c24 cm. 0aPerl (Computer program language)00801nam  22002778a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001300109042000800122050002600130082001800156100002000174245008800194250003200282260004100314263000900355300001100364650003700375650003600412650002600448700002500474700002400499fol05848297 IMchF20000524125727.0000518s2000    mau           001 0 eng    a   00041664   a1565924193  aDLCcDLC  apcc00aQA76.73.P22bG84 200000a005.2/7622211 aGuelich, Scott.10aCGI programming with Perl /cScott Guelich, Shishir Gundavaram & Gunther Birznieks.  a2nd ed., expanded & updated  aCambridge, Mass. :bO'Reilly,c2000.  a0006  ap. cm. 0aPerl (Computer program language) 0aCGI (Computer network protocol) 0aInternet programming.1 aGundavaram, Shishir.1 aBirznieks, Gunther.00665nam  22002298a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001300109042000800122050002700130082001700157111005200174245008600226250001200312260004100324263000900365300001100374650005000385fol05865950 IMchF20000615103017.0000612s2000    mau           100 0 eng    a   00055759   a0596000138  aDLCcDLC  apcc00aQA76.73.P22bP475 200000a005.13/32212 aPerl Conference 4.0d(2000 :cMonterey, Calif.)10aProceedings of the Perl Conference 4.0 :bJuly 17-20, 2000, Monterey, California.  a1st ed.  aCambridge, Mass. :bO'Reilly,c2000.  a0006  ap. cm. 0aPerl (Computer program language)vCongresses.00579nam  22002178a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001300109042000800122050002600130082001700156100002800173245006200201260004100263263000900304300001100313650003700324fol05865956 IMchF20000615102948.0000612s2000    mau           000 0 eng    a   00055770   a1565926099  aDLCcDLC  apcc00aQA76.73.P22bB43 200000a005.13/32211 aBlank-Edelman, David N.10aPerl for system administration /cDavid N. Blank-Edelman.  aCambridge, Mass. :bO'Reilly,c2000.  a0006  ap. cm. 0aPerl (Computer program language)00661nam  22002538a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001300109042000800122050002600130082001700156100001700173245006700190250001200257260004100269263000900310300001100319650003700330700002300367700001700390fol05865967 IMchF20000615102611.0000614s2000    mau           000 0 eng    a   00055799   a0596000278  aDLCcDLC  apcc00aQA76.73.P22bW35 200000a005.13/32211 aWall, Larry.10aProgramming Perl /cLarry Wall, Tom Christiansen & Jon Orwant.  a3rd ed.  aCambridge, Mass. :bO'Reilly,c2000.  a0007  ap. cm. 0aPerl (Computer program language)1 aChristiansen, Tom.1 aOrwant, Jon.00603cam  22002054a 4500001001300000003000600013005001700019008004100036010001700077020001500094040001800109042000800127050002600135082001700161100003200178245006000210260005700270300003300327650003700360fol05872355 IMchF20000706095105.0000315s1999    njua          001 0 eng    a   00500678   a013020868X  aDLCcDLCdDLC  apcc00aQA76.73.P22bL69 199900a005.13/32211 aLowe, Vincentq(Vincent D.)10aPerl programmer's interactive workbook /cVincent Lowe.  aUpper Saddle River, NJ :bPrentice Hall PTP,cc1999.  axx, 633 p. :bill. ;c23 cm. 0aPerl (Computer program language)00696nam  22002538a 4500001001300000003000600013005001700019008004100036010001700077020002800094040001300122042000800135050002600143082001700169100002600186245004400212260005100256263000900307300001100316500002000327650003700347650001700384650004100401fol05882032 IMchF20000707091904.0000630s2000    cau           001 0 eng    a   00058174   a0764547291 (alk. paper)  aDLCcDLC  apcc00aQA76.73.P22bF64 200000a005.13/32212 aFoster-Johnson, Eric.10aCross-platform Perl /cEric F. Johnson.  aFoster City, CA :bIDG Books Worldwide,c2000.  a0009  ap. cm.  aIncludes index. 0aPerl (Computer program language) 0aWeb servers. 0aCross-platform software development.

data/spec/data/one.dat

@@ -0,0 +1 @@
+00734njm a2200217uu 4500001001500000003000400015005001700019007001500036008004100051010001900092040001300111245006800124260002600192300005600218651004500274700005400319700005500373700003000428852005000458852000800508afc99990058366DLC20071104155141.9sd ummunniauub071103s1939    xxufmnne||||||||| u eng||  aafc99990058366  aDLCcDLC04aThe Texas rangerh[sound recording] /cSung by Beale D. Taylor.  aMedina, Texas,c1939.  a1 sound disc :banalog, 33 1/3 rpm, mono. ;c12 in. 0aMedinazTexaszUnited States of America.1 aLomax, John Avery, 1867-1948eRecording engineer.1 aLomax, Ruby T. (Ruby Terrill)eRecording engineer.1 aTaylor, Beale D.eSinger.  aAmerican Folklife Center, Library of Congress  aDLC