marcspec 0.8.1 → 1.0.0

data/CHANGES

@@ -1,3 +1,8 @@
+1.0.0
+  * Added constant specs (:constantValue=>'val' or :constantValue=>['array', 'of', 'values'])
+  * Arbitrarily decided this is version 1.0
+0.9.0
+  * Added ability to benchmark by calling ss.doc_from_marc(r, true) instead of just ss.doc_from_marc(r)
 0.8.1
   * Added some specs, squashed some bugs. In particular, make sure the Range passed to a ControlField
     makes sense (x..y, where x<=y and x > 0)

data/VERSION

@@ -1 +1 @@
-0.8.1
+1.0.0

data/lib/marcspec/constantspec.rb

@@ -0,0 +1,50 @@
+require 'marcspec/solrfieldspec'
+
+
+module MARCSpec
+  class ConstantSolrSpec < SolrFieldSpec
+    attr_accessor :constantValue
+    
+    #attr_accessor :solrField, :first, :map, :noMapKeyDefault, :marcfieldspecs, :default, :arity
+    
+    def initialize opts = {}
+      @solrField  = opts[:solrField]
+      @constantValue = opts[:constantValue]
+      @arity = 1
+      
+      # Check to make sure we didn't get anything else and warn if need be
+      [:firstOnly, :mapname, :noMapKeyDefault, :specs, :default, :module, :functionSymbol].each do |s|
+        if opts[s] 
+          raise ArgumentError, "#{s} is not a valid option for Constant spec (one with :constantValue defined)"
+        end
+      end
+    end
+    
+    def marc_values r, doc = {}
+      return @constantValue
+    end
+    
+    def == other
+      return @constantValue == other.constantValue
+    end
+    
+    def self.fromHash h
+      return self.new(h)
+    end
+    
+    def asPPString
+      s = StringIO.new
+      s.print "{\n :solrField=> "
+      PP.singleline_pp(@solrField, s)
+      s.print(",\n ")
+      s.print ":constantValue => "
+      PP.singleline_pp(@constantValue, s)
+      s.print "\n}"
+      return s.string
+    end
+    
+  end
+end
+      
+      
+    

data/lib/marcspec/specset.rb

@@ -1,6 +1,6 @@
 require 'jruby_streaming_update_solr_server'
 require 'marc4j4r'
-
+require 'benchmark'
     
 
 module MARCSpec
@@ -26,11 +26,12 @@ module MARCSpec
   end
 
   class SpecSet
-    attr_accessor :tmaps, :solrfieldspecs
+    attr_accessor :tmaps, :solrfieldspecs, :benchmarks
     
     def initialize
       @tmaps = {}
       @solrfieldspecs = []
+      @benchmarks = {}
     end
     
     def map name
@@ -57,6 +58,8 @@ module MARCSpec
       speclist.each do |spechash|
         if spechash[:module]
           solrspec = MARCSpec::CustomSolrSpec.fromHash(spechash)
+        elsif spechash[:constantValue]
+          solrspec = MARCSpec::ConstantSolrSpec.fromHash(spechash)
         else
           solrspec = MARCSpec::SolrFieldSpec.fromHash(spechash)
         end
@@ -77,6 +80,7 @@ module MARCSpec
     
     def add_spec solrfieldspec
       self.solrfieldspecs << solrfieldspec
+      @benchmarks[solrfieldspec.solrField] = Benchmark::Tms.new(0,0,0,0, 0, solrfieldspec.solrField)      
     end
     
     alias_method :<<, :add_spec
@@ -100,15 +104,39 @@ module MARCSpec
       end
     end
 
-    def doc_from_marc r
+    def fill_hashlike_from_marc_benchmark r, hashlike
+      @solrfieldspecs.each do |sfs|
+        @benchmarks[sfs.solrField] += Benchmark.measure do
+          if sfs.arity == 1
+            hashlike.add(sfs.solrField,sfs.marc_values(r, hashlike))
+          else 
+            vals = sfs.marc_values(r, hashlike)
+            (0..(sfs.arity - 1)).each do |i|
+              hashlike.add(sfs.solrField[i], vals[i])
+            end
+          end
+        end
+      end
+    end
+
+
+    def doc_from_marc r, timeit = false
       doc = SolrInputDocument.new
-      fill_hashlike_from_marc r, doc
+      if timeit
+        fill_hashlike_from_marc_benchmark r, doc
+      else 
+        fill_hashlike_from_marc r, doc
+      end
       return doc
     end      
     
-    def hash_from_marc r
+    def hash_from_marc r, timeit = false
       h = MARCSpec::MockSolrDoc.new
-      fill_hashlike_from_marc r, h
+      if timeit
+         fill_hashlike_from_marc_benchmark r, h
+       else 
+         fill_hashlike_from_marc r, h
+       end
       return h
     end
   end

data/lib/marcspec.rb

@@ -6,6 +6,7 @@ $LOG.level = Logger::WARN
 
 require "marcspec/customspec"
 require "marcspec/solrfieldspec"
+require "marcspec/constantspec"
 require "marcspec/kvmap"
 require "marcspec/multivaluemap"
 require "marcspec/specset"

data/spec/cachespot_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+# LEADER 00734njm a2200217uu 4500
+# 001    afc99990058366
+# 003    DLC
+# 005    20071104155141.9
+# 007    sd ummunniauub
+# 008    071103s1939    xxufmnne||||||||| u eng||
+# 010    $a afc99990058366
+# 040    $a DLC $c DLC
+# 245 04 $a The Texas ranger $h [sound recording] / $c Sung by Beale D. Taylor.
+# 260    $a Medina, Texas, $c 1939.
+# 300    $a 1 sound disc : $b analog, 33 1/3 rpm, mono. ; $c 12 in.
+# 651  0 $a Medina $z Texas $z United States of America.
+# 700 1  $a Lomax, John Avery, 1867-1948 $e Recording engineer.
+# 700 1  $a Lomax, Ruby T. (Ruby Terrill) $e Recording engineer.
+# 700 1  $a Taylor, Beale D. $e Singer.
+# 852    $a American Folklife Center, Library of Congress
+# 852    $a DLC
+
+describe "Cachespot" do
+  before do
+    @one = MARC4J4R::Reader.new("#{DIR}/data/one.dat").first
+  end
+  
+  it "should set a value and get it back" do
+    @one.cachespot['one'] = 1
+    @one.cachespot['one'].should.equal 1
+  end
+end

data/spec/controlfieldspec_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+# LEADER 00734njm a2200217uu 4500
+# 001    afc99990058366
+# 003    DLC
+# 005    20071104155141.9
+# 007    sd ummunniauub
+# 008    071103s1939    xxufmnne||||||||| u eng||
+# 010    $a afc99990058366
+# 040    $a DLC $c DLC
+# 245 04 $a The Texas ranger $h [sound recording] / $c Sung by Beale D. Taylor.
+# 260    $a Medina, Texas, $c 1939.
+# 300    $a 1 sound disc : $b analog, 33 1/3 rpm, mono. ; $c 12 in.
+# 651  0 $a Medina $z Texas $z United States of America.
+# 700 1  $a Lomax, John Avery, 1867-1948 $e Recording engineer.
+# 700 1  $a Lomax, Ruby T. (Ruby Terrill) $e Recording engineer.
+# 700 1  $a Taylor, Beale D. $e Singer.
+# 852    $a American Folklife Center, Library of Congress
+# 852    $a DLC
+
+describe "ControlFieldSpec" do
+
+  before do
+    @one = MARC4J4R::Reader.new("#{DIR}/data/one.dat").first
+    # @batch = MARC4J4R::Reader.new("#{DIR}/batch.dat").collect
+  end
+
+  # afc99990058366 # data
+  # 01234567890123 # index
+  it "gets a single full value" do
+    cfs = MARCSpec::ControlFieldSpec.new('001')
+    cfs.marc_values(@one).should.equal ["afc99990058366"]    
+  end
+  
+  it "gets a single character" do
+    cfs = MARCSpec::ControlFieldSpec.new('001', 10 )
+    cfs.marc_values(@one).should.equal ['8']
+  end
+  
+  it "gets a range of characters" do
+    cfs = MARCSpec::ControlFieldSpec.new('001', 6..10 )
+    cfs.marc_values(@one).should.equal ['90058']
+  end
+  
+  it "should round trip" do
+    cfs = MARCSpec::ControlFieldSpec.new('001', 6..10 )
+    cfs2 = MARCSpec::ControlFieldSpec.fromPPString(cfs.asPPString)
+    cfs.should.equal cfs2
+  end    
+end
+
+

data/spec/leaderspec_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+# LEADER 00734njm a2200217uu 4500
+# 001    afc99990058366
+# 003    DLC
+# 005    20071104155141.9
+# 007    sd ummunniauub
+# 008    071103s1939    xxufmnne||||||||| u eng||
+# 010    $a afc99990058366
+# 040    $a DLC $c DLC
+# 245 04 $a The Texas ranger $h [sound recording] / $c Sung by Beale D. Taylor.
+# 260    $a Medina, Texas, $c 1939.
+# 300    $a 1 sound disc : $b analog, 33 1/3 rpm, mono. ; $c 12 in.
+# 651  0 $a Medina $z Texas $z United States of America.
+# 700 1  $a Lomax, John Avery, 1867-1948 $e Recording engineer.
+# 700 1  $a Lomax, Ruby T. (Ruby Terrill) $e Recording engineer.
+# 700 1  $a Taylor, Beale D. $e Singer.
+# 852    $a American Folklife Center, Library of Congress
+# 852    $a DLC
+
+describe "LeaderSpec" do
+  before do
+    @one = MARC4J4R::Reader.new("#{DIR}/data/one.dat").first
+  end
+  
+  it "Works with full leader" do
+    cfs = MARCSpec::LeaderSpec.new('LDR')
+    cfs.marc_values(@one).should.equal @one.leader
+  end
+  
+  it "Works with substring of leader" do
+    cfs = MARCSpec::LeaderSpec.new('LDR', 3..5)
+    cfs.marc_values(@one).should.equal @one.leader[3..5]
+  end
+end

data/spec/solrfieldspec_spec.rb

@@ -213,4 +213,43 @@ describe "CustomSolrSpec" do
   end
   
 end
-  
+
+describe "ConstantSolrSpec" do
+  it "sets correct fields" do
+    c = MARCSpec::ConstantSolrSpec.new(:solrField=>"test", :constantValue=>"value")
+    c.solrField.should.equal 'test'
+    c.constantValue.should.equal 'value'
+  end
+  
+  it "allows array of values" do 
+    value = ['a', 'b', 'c']
+    c = MARCSpec::ConstantSolrSpec.new(:solrField=>"test", :constantValue=>value)
+    c.constantValue.should.equal value
+  end
+
+  bad = {
+    :firstOnly => true,
+    :default => 'default',
+    :noMapKeyDefault => 'nmd',
+    :mapname => 'map',
+    :specs => [],
+    :module => MARCSpec,
+    :functionSymbol => :test
+  }
+
+  bad.each do |k,v|
+    opts = {:solrField=>'test'}
+    opts[k] = v
+    it "raises ArgumentError if given invalid option #{k}" do
+      lambda{c = MARCSpec::ConstantSolrSpec.new(opts)}.should.raise ArgumentError
+    end
+  end
+  
+  it "should round trip" do
+    c = MARCSpec::ConstantSolrSpec.new(:solrField=>"test", :constantValue=>"value")
+    s = StringIO.new
+    s.puts(c.asPPString)
+    d = MARCSpec::ConstantSolrSpec.fromPPString(s.string)
+    c.should.equal d
+  end
+end

data/spec/specset_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 
+positive = lambda{|x| x > 0}
+
 module A
   module B
     def self.titleUp doc, r, codes=nil
@@ -46,22 +48,43 @@ describe "SpecSet Basics" do
         ["651",  "z"],
        ]
       },
-     {:solrField=>'title', :specs=>[['245']]},
+      {
+       :solrField=>'title', 
+       :specs=>[['245']]
+      },
       {
         :solrField => 'titleA',
         :specs => [['245',  'a']]
+      },
+      {
+        :solrField => 'constantField',
+        :constantValue => ['A', 'B']
       }
     ]
+    
+    @ss = MARCSpec::SpecSet.new
+    @ss.buildSpecsFromList(@speclist)
+    @h = @ss.hash_from_marc @one
+    
   end
   
-  it "should build from a list" do 
-    ss = MARCSpec::SpecSet.new
-    ss.buildSpecsFromList(@speclist)
-    ss.solrfieldspecs.size.should.equal 3
-    h = ss.hash_from_marc @one
-    h['places'].sort.should.equal @places.sort
-    h['title'].should.equal @title
-    h['titleA'].should.equal @titleA
+  it "should get all the specs" do
+    @ss.solrfieldspecs.size.should.equal 4
+  end
+  
+  it "gets the places field" do
+    @h['places'].sort.should.equal @places.sort
+  end
+  
+  correct = {
+    'title' => @title,
+    'titleA' => @titleA,
+    'constantField' => ['A', 'B']
+  }
+  correct.each do |k,v|
+    it "gets correct value for #{k}" do
+      @h[k].should.equal v
+    end
   end
   
   it "allows customs that reference previous work" do
@@ -96,7 +119,49 @@ describe "SpecSet Basics" do
     h['letters'].should.equal ['a', 'b']
   end
 end
-   
+
+
+
+describe "Specset Benchmarking" do
+  before do
+    @reader = MARC4J4R::Reader.new("#{DIR}/data/batch.dat")    
+    @speclist = [
+      {
+       :solrField=>'title', 
+       :specs=>[['245']]
+      },
+      {
+       :solrField=> "places",
+       :specs => [
+        ["260",  "a"],
+        ["651",  "a"],
+        ["651",  "z"],
+       ]
+      },      
+      {
+        :solrField => 'titleA',
+        :specs => [['245',  'a']]
+      }
+    ]
+    
+    @ss = MARCSpec::SpecSet.new
+    @ss.buildSpecsFromList(@speclist)    
+  end
+  
+  it "should benchmark" do
+    @reader.each do |r|
+      h = @ss.hash_from_marc(r, true)
+    end
+    @ss.solrfieldspecs.each do |sfs|
+      @ss.benchmarks[sfs.solrField].real.should.be positive
+    end
+    
+    # @ss.benchmarks.each do |k,v|
+    #   puts "%-10s %s" % [k + ':', v.to_s]
+    # end
+  end
+end
+    
 
 
 

data/spec/variablefieldspec_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+# LEADER 00734njm a2200217uu 4500
+# 001    afc99990058366
+# 003    DLC
+# 005    20071104155141.9
+# 007    sd ummunniauub
+# 008    071103s1939    xxufmnne||||||||| u eng||
+# 010    $a afc99990058366
+# 040    $a DLC $c DLC
+# 245 04 $a The Texas ranger $h [sound recording] / $c Sung by Beale D. Taylor.
+# 260    $a Medina, Texas, $c 1939.
+# 300    $a 1 sound disc : $b analog, 33 1/3 rpm, mono. ; $c 12 in.
+# 651  0 $a Medina $z Texas $z United States of America.
+# 700 1  $a Lomax, John Avery, 1867-1948 $e Recording engineer.
+# 700 1  $a Lomax, Ruby T. (Ruby Terrill) $e Recording engineer.
+# 700 1  $a Taylor, Beale D. $e Singer.
+# 852    $a American Folklife Center, Library of Congress
+# 852    $a DLC
+
+describe "VariableFieldSpec" do
+  before do
+    @one = MARC4J4R::Reader.new("#{DIR}/data/one.dat").first
+    # @batch = MARC4J4R::Reader.new("#{DIR}/batch.dat").collect
+  end
+
+  it "Should get a whole field separated by spaces" do
+    dfs = MARCSpec::VariableFieldSpec.new('260')
+    dfs.marc_values(@one).should.equal ["Medina, Texas, 1939."]
+  end
+
+  it "Should get just the $a" do
+    dfs = MARCSpec::VariableFieldSpec.new('260', 'a')
+    dfs.marc_values(@one).should.equal ["Medina, Texas,"]
+  end
+  
+  it "should return separate values for repeated subfields if only one code is specified" do
+    dfs = MARCSpec::VariableFieldSpec.new('651', 'z')
+    dfs.marc_values(@one).sort.should.equal ['Texas', 'United States of America.']
+  end
+  
+  it "Should get all fields via several equal routes" do
+    a = MARCSpec::VariableFieldSpec.new('260').marc_values(@one)
+    ac =  MARCSpec::VariableFieldSpec.new('260', ['a', 'c']).marc_values(@one)
+    ca =  MARCSpec::VariableFieldSpec.new('260', ['c', 'a']).marc_values(@one)
+    ca2 = MARCSpec::VariableFieldSpec.new('260', 'ca').marc_values(@one)
+    allrange = MARCSpec::VariableFieldSpec.new('260', 'a'..'z').marc_values(@one)
+    a.should.equal ac
+    ac.should.equal ca
+    ca.should.equal allrange
+  end
+  
+  it "should get all three 700a's" do
+    a = MARCSpec::VariableFieldSpec.new('700', 'a').marc_values(@one)
+    a.should.equal ["Lomax, John Avery, 1867-1948", "Lomax, Ruby T. (Ruby Terrill)", "Taylor, Beale D."]
+  end
+  
+  it "should round trip" do 
+    ac =  MARCSpec::VariableFieldSpec.new('260', ['a', 'c'])
+    ac2 = MARCSpec::VariableFieldSpec.fromPPString(ac.asPPString)
+    ac.should.equal ac2
+  end
+  
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: marcspec
 version: !ruby/object:Gem::Version 
-  hash: 61
+  hash: 23
   prerelease: false
   segments: 
-  - 0
-  - 8
   - 1
-  version: 0.8.1
+  - 0
+  - 0
+  version: 1.0.0
 platform: ruby
 authors: 
 - BillDueber
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-04 00:00:00 -04:00
+date: 2010-09-14 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -96,6 +96,7 @@ files:
 - Rakefile
 - VERSION
 - lib/marcspec.rb
+- lib/marcspec/constantspec.rb
 - lib/marcspec/controlfieldspec.rb
 - lib/marcspec/customspec.rb
 - lib/marcspec/kvmap.rb
@@ -106,6 +107,8 @@ files:
 - lib/marcspec/solrfieldspec.rb
 - lib/marcspec/specset.rb
 - lib/marcspec/variablefieldspec.rb
+- spec/cachespot_spec.rb
+- spec/controlfieldspec_spec.rb
 - spec/data/batch.dat
 - spec/data/one.dat
 - spec/data/umich/translation_maps/area_map.properties
@@ -122,11 +125,13 @@ files:
 - spec/data/umich/translation_maps/library_map.properties
 - spec/data/umich/translation_maps/location_map.properties
 - spec/data/umich/umich_index.properties
+- spec/leaderspec_spec.rb
 - spec/maps_spec.rb
 - spec/marcfieldspecs_spec.rb
 - spec/solrfieldspec_spec.rb
 - spec/spec_helper.rb
 - spec/specset_spec.rb
+- spec/variablefieldspec_spec.rb
 has_rdoc: true
 homepage: http://github.com/billdueber/marcspec
 licenses: []
@@ -162,8 +167,12 @@ signing_key:
 specification_version: 3
 summary: Extract data from MARC records and send to Solr
 test_files: 
+- spec/cachespot_spec.rb
+- spec/controlfieldspec_spec.rb
+- spec/leaderspec_spec.rb
 - spec/maps_spec.rb
 - spec/marcfieldspecs_spec.rb
 - spec/solrfieldspec_spec.rb
 - spec/spec_helper.rb
 - spec/specset_spec.rb
+- spec/variablefieldspec_spec.rb