zcc 0.0.3 → 0.1.0

data/lib/zcc/query.rb

@@ -0,0 +1,183 @@
+module ZCC
+
+  class Query
+    attr_reader :term, :zservers, :type, :zsearch
+    
+    def initialize(term, zservers)
+      self.term = term #=> string of original query post any filtering
+      #puts "term: " + self.term
+      @zservers = []
+      @zservers << zservers #=>array of Zserver objects
+      self.zservers.compact!
+      self.zservers.flatten!
+      #puts self.inspect
+      #puts self.zservers[0].to_s
+    end
+    
+    def term=(term)
+      if term =~ /\d+-\d+/
+        @term = lccn_conversion(term)
+        @type = 'lccn'
+        #puts "converted lccn to #{@term}"
+        @zsearch = "@attr 1=9 #{@term}"
+      elsif term =~ /\d+[X||\d]/ and !(term.match( /[a-wyz]/i ))
+        #puts "Searching for ISBN: #{term}"
+        @term = term
+        @type = 'isbn'
+        @zsearch = "@attr 1=7 #{@term}"
+      else ( term[/[a-z]/i] ) #-- This check for a string could be better!
+        #puts "searching for title:#{term}"
+        if term.match(/ :au /)
+          term = term.split(" :au ")
+          puts "Author and title then"
+          @term = term[0]
+          @zsearch = "@and @attr 1=4 \"#{@term}\" @attr 1=1 \"#{term[1]} \"" #was: "@attr 1=4 \"'#{@term}'\""
+        else
+          @term = term
+          @zsearch = "@attr 1=4 \"#{@term}\"" #was: "@attr 1=4 \"'#{@term}'\""
+        end
+        @type = 'title'
+        
+        #puts self.term
+        #puts self.type
+        #puts self.zsearch
+        
+      end
+    end
+    
+    #This is the main search logic of the whole shebang. Aliased as 'search'.
+    #The method on a query object and is passed the number of records from each host to present to the user. Default number of records is 10.
+    #Still need to work on a way to get options in so this might change.
+    #Currently only MARC21 is supported.
+    def zoom(show = 10)
+      result_set = ResultSet.new(self)
+      #puts zservers.inspect
+      search_threads=[]
+      puts zservers.size.to_s + " z-servers in group " + zservers[0].group.to_s
+      z = 0
+      self.zservers.each do |server|
+          z += 1
+          #zservers are made up of...
+          #puts server.to_s
+        search_threads << Thread.new(server, z) do |myserver, myz|
+          begin
+            conn = ZOOM::Connection.new
+            conn.connect(myserver.host, server.port) #do |conn 
+            conn.set_option('charset', 'UTF-8')
+            conn.preferred_record_syntax = 'MARC21'
+            conn.database_name = myserver.database
+            puts "#{myz} Searching:          #{myserver.to_s} | #{self.zsearch}"
+            rset = conn.search(self.zsearch)
+            say(myz.to_s.headline + " Finished searching: #{myserver.to_s} | rset.size: " + "#{rset.size}".red.bold)
+            rset_recs = rset[0, show]
+            #puts "rset_recs in query.search: " 
+            #puts rset_recs
+            i = 0
+            rset_recs.each do |rec|
+              #puts myserver.to_s
+              #puts rec
+              #puts
+              marc_record = ZCC.convert_char(rec)
+              #puts "gets past character conversion"
+              #puts "-------------\n"
+              #puts marc_record
+              #puts "---------------\n"
+              zcc_record = Record.new(marc_record, myserver)
+              result_set.ingest(zcc_record)
+              puts "#{myz} record #{i} from       #{myserver}..."
+              i += 1
+            end
+          rescue Exception => e
+            zerror_log("dead thread: " + myserver.to_s + " | " + e)
+            puts "\a#{myz}!!!!!!!! Thread died #{myserver} !!!!!"
+          end
+          #puts "end: #{Thread.list}"
+          #puts "Results processed from:  #{myserver.to_s}"
+        end
+        
+      end
+      search_threads.each{|thread| thread.join}
+      return result_set
+    end
+    
+    alias search zoom
+    def zerror_log error
+      File.open("#{File.expand_path("~")}/.zcc/zerror_log", "a+") do |f|
+        f.write error + "\n"
+      end
+    end
+    
+  end
+  
+  
+  def convert_char rsetrec
+    rec = MARC::Record.new_from_marc(rsetrec.raw)
+    #puts "initial rec" + rec.to_s
+    ldr9 = rec.leader[9, 1]
+    return_rec = ''
+    #puts "gets to creating dummy record."
+    if ldr9 == ' '
+      #return_rec = MARC::Record.new_from_marc(rsetrec.raw('MARC-8', 'UTF-8')) #This does NOT work
+      return_rec = MARC::XMLReader.new(StringIO.new(rsetrec.xml('MARC-8', 'UTF-8'))).to_a
+      return_rec = return_rec[0]
+      return_rec.leader[9,1] = 'a'
+      #puts "return_rec" + return_rec.to_s
+    elsif ldr9 == 'a'
+      #puts "already unicode"
+      return_rec = rec
+    else
+      raise "Invalid value in leader 9 for MARC21"
+    end
+    return_rec
+  end
+  
+  def lccn_conversion lccn
+    split_lccn = lccn.split('-')
+    year_len = split_lccn[0].length
+    serial_len = split_lccn[1].length
+    start_length = year_len + serial_len
+    if year_len == 2
+      final_lccn = lccn.gsub('-', "#{'0' * (8 - start_length)}")
+    elsif year_len == 4
+      final_lccn = lccn.gsub('-', "#{'0' * (10 - start_length)}")
+    end
+    final_lccn
+  end
+  
+    #++ compare() is a method to print out a comparison of two MARC records tag by tag (not by subfields).
+  # A match between lines is denoted with a 'm'. If there are differences between the records,
+  # the object that recieves the compare call is denoted with a '+' and the object passed in 
+  # parens is denoted with '-'. 
+  def compare_marc(orig, rec2)
+    puts "+ = #{orig.zserver.to_s}".bold
+    puts "- = #{rec2.zserver.to_s}".bold
+    orig = orig.marc
+    rec2 = rec2.marc
+    
+    for ft in ('000'..'999')
+      fields_original = orig.find_all {|f| f.tag == ft}
+      fields_record2 = rec2.find_all {|f| f.tag == ft}
+      fields_orig = []
+      fields_rec2 = []
+
+      fields_original.each {|f| fields_orig << f.to_s}
+      fields_record2.each {|f| fields_rec2 << f.to_s}
+
+      matches = fields_orig & fields_rec2
+      matches.each { |f| puts "m".bold + " #{f}" } if matches			
+
+      fields_orig -= matches
+      fields_orig.each {|f| puts "+".bold + " #{f}"} if fields_orig
+
+      fields_rec2 -= matches
+      fields_rec2.each {|f| puts "-".bold + " #{f}"} if fields_rec2
+    end
+  end
+  
+  def blank_field_prompt(field, subfield)
+    #puts "subfield: #{subfield}"
+    value = ask("\nYour MARC record does not contain #{field}#{subfield}.\nEnter the intended value for #{field}#{subfield} or hit ENTER to leave blank\n#{field}#{subfield} > ")
+    value
+  end
+  
+end

data/lib/zcc/record.rb

@@ -0,0 +1,265 @@
+module ZCC
+  
+  class Record     
+  attr_accessor :selected, :marc, :authorities, :rank
+  attr_reader :zserver
+    def initialize(marc, zserver, selected = false)
+      @marc = marc
+      @zserver = zserver
+      @selected = selected
+      @rank = 0
+    end
+    
+    #def authorities<< (authority)
+    #end
+    
+    def to_s
+      full_string = "\n---------------RECORD----------------\nzserver: " + zserver.to_s + "\n" +  marc.to_s +  "---------------RECORD----------------\n"
+      full_string
+    end
+    
+    def title 
+      self.marc['245']['a']
+    end 
+    
+    def author 
+    end #1XX?
+    
+    def extent 
+      self.marc['260']['a']
+    end 
+    
+    def year_260
+      if self.marc['260'] && self.marc['260']['c']
+        date = self.marc['260']['c'].dup
+        date.gsub!('[', '')
+        date.gsub!(']', '')
+        date.gsub!('c','')
+        date
+      else
+        date = '0'
+        date
+      end
+    end
+    
+    def years_fixed 
+    end
+    
+    def save(format)
+    end #save MARC record to file in proper format (MARC, MARCXML)
+    
+    
+    
+    # linter depends on Perl's MARC::Lint and hence Perl's MARC::Record
+    # Use cpan to install them.
+    def linter
+      rec = self.marc
+      xml_rec = (rec.to_xml).to_s
+      if LINTER == false
+        puts 'You do not have the Perl MARC::Lint module installed or have disabled this feature.'
+        return
+      end
+      contents = `perl "#{File.expand_path("~")}"/.zcc/linter.pl "#{xml_rec}"`
+      if contents.empty?
+        puts "there were no errors detected by the linter."
+      else
+        puts contents	
+      end
+      if rec.leader[18,1] == 'a'
+        puts "Leader indicates: AACR"
+      elsif rec.leader[18,1] == 'i'
+        puts "Leader indicates: ISBD"
+      else
+        puts "Leader indicates NOT AACR nor ISBD."
+      end
+    end
+ 
+     #++ The local_script method is for automating changes to the record before saving it. See the main script to turn on this feature. See zoomer.yaml for instructions on creating a script for your purposes.
+  def local_script(script)
+    #print $clear_code
+    record = self.marc
+
+    #--creating my procs for creating fields and appending subfields
+    # these two should probably be moved to methods
+    create_datafield = proc{|record, field, i1, i2| puts "Creating datafield: #{field}"; record.append(MARC::DataField.new(field.to_s, i1.to_s, i2.to_s)); puts "****", record, "***" if $testing;}
+
+    append_subfield = proc{|record, field, subfield, val| 
+      field = field.to_s
+      subfield = subfield.to_s
+      value = val.dup
+      value = value.to_s
+      if value.include?('proc ')
+        #I've had problems with this bit but it seems to work now
+        value = value.sub(/^proc /, '')
+        real_value = eval("#{$procs[value]}") 
+        next if real_value == nil                
+        record[field].append(MARC::Subfield.new(subfield, real_value))
+      elsif
+        record[field].append(MARC::Subfield.new(subfield, value))
+      end
+    }
+
+    script.each do |single_script|
+      puts "--------------------------------------" if $testing
+      puts single_script.join(', ') if $testing
+      op, field, subfield, filler= single_script
+      field = field.to_s
+      subfield = subfield.to_s
+      operation = op.dup
+      if operation.include?('proc ')
+        operation = operation.dup.sub(/^proc /, '')
+        eval("#{$procs[operation]}")
+      elsif operation == 'create-field'
+        create_datafield.call(record, field, subfield, filler)
+      elsif operation == 'append-subfield'
+        append_subfield.call(record, field, subfield, filler)
+      elsif operation == 'prompt'
+        field_data = ask("What do you want to go into the #{field}#{subfield} #{filler}? ")
+        
+        next if field_data.empty?
+        m_fields = record.find_all{|x| x.tag == field}
+        if m_fields.empty?
+          puts "m_fields is empty!!"
+          create_datafield.call(record, field, ' ', ' ')
+          m_fields = record.find_all{|x| x.tag == field}
+        end
+        m_fields.each {|field| field.append(MARC::Subfield.new(subfield, field_data))}
+      elsif operation == 'remove'
+        to_remove = record.find_all {|f|  f.tag =~ Regexp.new( field.gsub('X','.'))}
+        to_remove.each do |remove|
+          record.fields.delete(remove)
+        end
+      elsif operation == 'sort-tags' ##This doesn't work right now
+        puts "sorting by tag"
+        puts record , "################"
+        record = record.sort_by{| field | field.tag}
+        puts record
+        STDIN.gets
+      else
+        puts "there's nothing for that yet in local_script"
+      end
+      puts record if $testing
+    end
+
+    record
+  end
+
+    
+  
+    def edit
+      puts "Subfield Editing"
+      puts "To denote which subfield you want to edit put it in the form of 245a.\nIn the case of repeating fields or subfields you will be prompted\nto choose the one you wish to edit. "
+      continue = true
+      while continue
+	      fs = ask("What field and subfield would you like to edit (in form 245a) or quit?\n>   "){|q| q.readline = true}
+        if fs == 'q' || fs == 'quit'
+          continue = false
+        next
+        end
+        if !(fs =~/\d\d\d\w/ )
+          puts "That's not a valid value"
+          next
+        end
+        self.change_subfield(fs)
+      end
+    end
+
+    #edit_subfield is passed to a MARC::Record object and give a subfield
+    def change_subfield(fs)
+      field_subfield = subfield_parse(fs)
+      field = field_subfield[0]
+      subfield = field_subfield[1]
+
+      fields = self.marc.find_all {|f| f.tag == field}
+      subfields = []
+      for field in fields
+	      subfields << field.find_all {|s| s.code == subfield}
+      end
+      subfields.flatten!
+      if subfields.length > 1
+        i = 0
+        for subfield in subfields
+	        puts "#{i}\t#{subfield}"
+          i += 1
+        end
+	      num_to_change = ask("Which subfield do you want to change?"){|q| q.readline = true}
+	      subfield_to_change = subfields[num_to_change.to_i]
+      elsif subfields.length == 1
+        subfield_to_change = subfields[0]
+      elsif subfields.empty?
+        puts "No such subfield!"
+	      return nil
+      end
+      
+
+      puts self.marc; puts
+
+      print "Currently:\t"
+      puts subfield_to_change.value 
+      edit = ask("Your edit:\t"){|q| q.readline = true}
+      subfield_to_change.value = edit
+      puts; puts self.marc; puts
+
+    end
+
+    #subfield_parse method for getting from '245a' to ['245','a']
+    def subfield_parse(combined)
+      field_subfield = []
+      field_subfield << combined[0, 3]
+      field_subfield << combined[3,1]
+    end
+  
+  
+    
+   # To use marc_to_csv it must be passed a csv template in the order of the fields.
+  # See the zoomer.yaml file for instructions on creating a template.
+  # See the main script for turning this feature on.
+  def marc_to_csv(template)
+    values = []
+    template.each do |template|
+      field, subfield, leng = template
+      field = field.to_s
+      subfield = subfield.to_s
+      if field == 'prompt'
+        value = ask("prompt: please enter the #{field} #{subfield}") #--subfield here is a label for the prompt
+        
+        value = "\"#{value}\","
+        values << value
+      elsif field.length == 3 and field.match('\d')
+        fields = self.marc.find_all { | f |  f.tag =~ Regexp.new( field.gsub('X','.'))}
+        value = ''
+        if fields.empty?
+          #puts "oh, no!"
+          value = blank_field_prompt(field, subfield)
+          #eval(blank_field_prompt)
+        else
+          fields.each do |f|
+            if f[subfield]
+              value += f[subfield] + "|"
+            else
+              value = blank_field_prompt(field, subfield)
+            end
+          end
+        end
+        value.sub!(/\|$/, '')
+        value = value[0, leng] if leng
+        #puts "value: #{value}"
+        value = "\"#{value}\","
+        values << value
+
+      else
+        puts "this is not a valid value for marc_to_csv"
+      end
+    end
+    #puts values
+    values[-1].sub!(/\,$/, '')
+    values = values.to_s
+    values += "\n"
+  end
+
+ 
+
+    
+  end
+
+end

data/lib/zcc/resultset.rb

@@ -0,0 +1,169 @@
+module ZCC  
+  class ResultSet
+    include Enumerable
+    
+    attr_accessor :query, :sort_by, :index_start, :index_pos, :records, :rank
+    
+    #All values of initialize are optional, though you'll want to supply it with a query object if you intend on getting records into your set through a z39.50 search. index_start and index_end will be used for the TUI display.
+    def initialize(query_object=nil, sort_by='title', index_start=0, index_pos=4 ) 
+      @query = query_object #query object
+      
+      @sort_by = sort_by
+      @index_start = index_start
+      @index_pos = index_pos
+      #puts self.sort_by
+      #puts self.index_start
+      @records = []
+      #puts self.records.class
+    end
+    
+    #Pretty prints the result set object. Nests pretty printed record objects within.
+    def to_s
+      full_string =  "-------RESULT SET--------------\n" + "Result set has #{self.records.size} records."
+      self.records.each do |record|
+        full_string << record.to_s
+      end
+      full_string += "-------RESULT SET--------------\n"
+    end
+
+    # Method to add records only to a result set.
+    def ingest record
+      self.records << record
+    end
+    
+    # appends a result set to another.
+    # Replaces the query and sort_by instance variables with the new ones.
+    
+    
+    #Removes unselected records from the result set. Uses the selected instance variable to check for true or false.
+    def remove_unselected!
+      self.records.each_index do |i|
+        if self.records[i].nil?
+        else
+          self.records[i] = nil unless self.records[i].selected
+        end
+      end
+      self.records.flatten!
+      self.records.compact!
+      self.records.uniq!
+    end
+    
+    #Number of records in the result set
+    def size
+      self.records.length
+    end 
+    
+    alias length size
+
+    #returns number of ZCC::Records with @selected set to true
+    def selected_size
+      selected_records = self.find_all{|record| record.selected unless record == nil}
+      selected_records.length
+    end 
+    
+    alias selected_length selected_size
+    
+    # This allows for Enumerable mixin.
+    def each 
+      for record in @records
+        yield record
+      end
+    end
+    
+    def sort_by_title!
+      self.records.compact!
+      self.records.flatten!
+      self.records = self.records.sort_by{|r| r.marc['245']['a']}
+    end
+
+    def sort_by_date!
+      self.remove_nil!
+      self.records = self.records.sort_by{ |r| r.year_260}         
+    end
+
+    def sort_by_subfield! sf
+      self.remove_nil!
+      field, subfield = sf[0,3], sf[3,10]
+      subfield = 'a' if subfield == ''
+      nil_subfields = []
+      self.records.each_index do |i| 
+	      unless self.records[i].marc[field] && self.records[i].marc[field][subfield]
+          nil_subfields << self.records[i]
+          self.records[i] = nil
+	      end
+      end
+      self.records.compact!
+      begin
+        self.records = self.records.sort_by{ |r| r.marc[field][subfield]}
+      rescue Exception => e
+        puts e
+        say_help("There was an error.\nYour records have not been sorted.")
+      end
+      self.records << nil_subfields
+      self.remove_nil!
+    end
+
+    def sort_by_standard!
+      self.remove_nil!
+      self.records = self.records.sort_by{|r| r.marc.leader[18]}
+      self.records.reverse
+    end
+
+    # Very simple relevancy ranking for title searches
+    def rank_by_relevance!
+      rank = 0
+      #raise NotImplementedError
+      #unless self.query.type == 'title'
+      #  raise "Relevancy ranking only works with titles (and not even them yet)"
+      #end
+      self.rank
+      term = self.query.term
+      re_term = Regexp.new("#{term}", true)
+      re_term2 = Regexp.new("\^#{term}", true)
+      self.records.each do |rec|
+        #puts re_term
+        #puts rec.marc['245']['a'] =~ re_term
+        rec.rank += 5 if rec.marc['245']['a'] =~ re_term
+        rec.rank += 10 if rec.marc['245']['a'] =~  re_term2
+      end
+      self.records = self.records.sort_by{|record| record.rank}
+      self.records.reverse!
+    end
+
+    def remove_nil!
+      self.records.compact!
+      self.records.flatten!
+    end
+    
+    def hits_per_source 
+    end #=> hash{zurl=>[rec, rec],zurl=>[rec, rec]
+    
+    def []
+      
+    
+    end # Record at that position
+    
+    def empty?
+      if self.records.size == 0
+        return true
+      elsif self.records.size > 0
+        return false
+      end
+    end #returns true if the @ZCCRecords array is empty or only nil values
+    
+    #def find_all
+      #self.find_all {|record| record.selected}
+    #end
+
+    def <<(result_set)
+      #puts record
+      self.records << result_set.records
+      self.query = result_set.query
+      self.sort_by = result_set.sort_by      
+      self.records.flatten!
+      #puts self.records.inspect
+    end
+
+    
+  end
+end