zcc 0.1.0 → 0.2.0

data/History.txt

@@ -1,3 +1,19 @@
+== 0.2.0 Wed Nov 21 14:43:00 EST 2007
+* This version is incompatible with previous versions.
+* major changes:
+  * added full record editing using yaz-marcdump command line utility and
+    the plain text editor of your choice
+  * uses extended services of zoom-0.4.1 to save records to zebra database
+  * ability added to configure and pick from various different script to
+    be run over records
+* minor changes:
+  * menu options and how choices work have changed
+  * shows whole record if only one is fetched
+  * added config option to automatically start and 
+    kill a zebra server
+
+
+
 == 0.1.0
 * The version is incompatible with previous versions.
 * major changes:

data/Manifest.txt

@@ -6,10 +6,12 @@ Rakefile
 bin/zcc
 examples/linter.pl
 examples/zcc.yaml
+examples/zebra/accessfile
 examples/zebra/key/remove_me
 examples/zebra/lock/remove_me
-examples/zebra/records/0.mrc
+examples/zebra/records/0.xml
 examples/zebra/register/remove_me
+examples/zebra/shadow/remove_me
 examples/zebra/tab/bib1.abs
 examples/zebra/tab/bib1.att
 examples/zebra/tab/default.idx
@@ -17,13 +19,13 @@ examples/zebra/tab/explain.abs
 examples/zebra/tab/explain.att
 examples/zebra/tab/explain.tag
 examples/zebra/tab/gils.att
-examples/zebra/tab/marc21.abs
 examples/zebra/tab/numeric.chr
 examples/zebra/tab/record.abs
 examples/zebra/tab/sort-string-utf.chr
 examples/zebra/tab/tagsetm.tag
 examples/zebra/tab/usmarc.mar
 examples/zebra/tab/word-phrase-utf.chr
+examples/zebra/zcc_passw
 examples/zebra/zebra.cfg
 examples/zoomer-iterator.txt
 examples/zservers.yaml
@@ -33,9 +35,7 @@ lib/zcc/cli_display.rb
 lib/zcc/query.rb
 lib/zcc/record.rb
 lib/zcc/resultset.rb
-lib/zcc/subfieldeditor.rb
 lib/zcc/version.rb
-lib/zcc/zoomer.rb
 lib/zcc/zserver.rb
 scripts/txt2html
 setup.rb

data/Rakefile

@@ -73,7 +73,7 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
   p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
   p.extra_deps = [
   ['marc','>=0.1.6'],
-  ['zoom', '>=0.3.0'],
+  ['zoom', '>=0.4.1'],
   ['unicode','>=0.1'],
   ['highline', '>=1.2.9'],
   ['term-ansicolor', '>=1.0.2']

data/bin/zcc

@@ -1,228 +1,352 @@
-#!/usr/bin/ruby -w
-## Jason Ronallo
-# July, 2007
-#zcc-new
-
-require 'rubygems'
-require 'zoom'
-require 'marc'
-require 'highline/import'
-require 'term/ansicolor'
-#include Term::ANSIColor
-
-require 'stringio'
-require 'yaml'
-require 'tempfile'
-
-$KCODE = 'u'
-require 'jcode'
-require 'unicode'
-
-require 'zcc'
-include ZCC
-
-termsize = HighLine::SystemExtensions.terminal_size
-$term_width = termsize[0]
-$term_height = termsize[1]
-$clear_code = %x{clear}
-HighLine.track_eof = false
-
-##Setting up testing parameter
-$testing = ARGV[0]
-$testing = nil unless $testing == 'v' or $testing == 't'
-file_mode = true if ARGV[0] == 'file'
-puts "We're testing " if $testing 
-STDIN.gets if $testing
-
-#reading in config from yaml file
-yamlfile = YAML.load_file( "#{File.expand_path("~")}/.zcc/zcc.yaml" )
-zservers = yamlfile['zservers']
-
-if zservers[0][0] =~ /Include/
-  filename = zservers[0][0][8, 99]
-  contents = YAML.load_file("#{File.expand_path("~")}/.zcc/#{filename}")
-  contents.each{|c| zservers.push(c)}
-  zservers[0] = nil
-end
-zservers.compact!
-
-labels = yamlfile['labels']
-scripting = yamlfile['scripting']
-$procs = yamlfile['procs']
-
-TO_SHOW = yamlfile['to_show']
-SHOW_PER_PAGE = yamlfile['show_per_page']
-FIELDS_TO_SHOW = yamlfile['fields_to_show']
-HOW_TO_SAVE = yamlfile['save_filename']
-TIMESTAMP = Time.now.strftime("%Y%m%d%H%M%S")
-SCRIPTING = yamlfile['scripting_on']
-EDITING = yamlfile['subfield_editing']
-CSV = yamlfile['csv_on']
-LINTER = yamlfile['linter_on']
-FORMAT_TO_SAVE = yamlfile['save_record_syntax']
-
-print $clear_code
-#printed once when starting the program.
-say("Zcc, Copyright (C) 2007 Jason Ronallo\nZcc comes with ABSOLUTELY NO WARRANTY.\nZcc is released under the terms of the GPL v.2 or later.\nSee LICENSE for full information.\n".bold)
-say("Simply enter a title, LCCN or ISBN and hit enter.\nTitle: enter the title exactly as it is on the source\n   for better relevancy ranking by title.\nLCCN: Include the dash, like 81-21962\nEnter ISBNs without dashes.\nNot all servers accept it, but you can also do a narrowing search\nby including an author's name after the title like so:\nFinnegans Wake :au Joyce")
-say("For more help see: http://zcc.rubyforge.org/zcc.html")
-
-zserver_objects = []
-zservers.each do |server|
-  zserver_objects << Zserver.new(server[0], server[1], server[2], server[3])
-end
-
-server_h = Hash.new {|hash, key| hash[key] = [] }
-zserver_objects.each {|z| server_h[z.group] << z}
-
-groups = []
-server_h.each_key{|k| groups << k}
-groups.uniq!
-groups.sort!
-
-loop {
-  search_term = ask("\nEnter search. [ISBN, title, LCCN (with dash), or :q to quit]\nsearch> ".boldz, String){ |q| q.readline = true }
-    exit if search_term == ':q'
-    
-  taken = ResultSet.new  
-  groups.each do |group|
-    print $clear_code
-    servers = server_h[group]
-    query = Query.new(search_term, servers)
-    results = query.search(TO_SHOW)
-    results.index_pos = SHOW_PER_PAGE - 1
-    if results.records.empty?
-      puts "There are no results from group #{group} servers!\n\n\n"
-      another_group = ask("Do you want to continue searcing the next group of servers? [y], anything  ".boldz)
-      if another_group == 'y' || another_group == ''
-        next
-      else
-        print $clear_code
-        break
-      end
-    end
-    results.rank_by_relevance!
-    return_value = zcc_select_good_marc(results, 'multi')
-    taken << results
-    break if return_value == 'done'    
-  end
-  
-      taken.remove_unselected!
-      next if taken.empty?
-      continue = true
-      print $clear_code
-      say("\aThis is your combined result set.\nWe will winnow until you choose ".white.bold.on_black + "'done'".red.bold + "\nor you unselect all records.".white.bold.on_black)
-      sleep 3
-      while continue
-       return_value = zcc_select_good_marc(taken, 'multi')
-       continue = false if taken.selected_length == 0
-       continue = false if return_value == 'done'
-      end
-      taken.remove_unselected!
-  unless taken.length == 0
-    puts "Here are the records you've selected for processing:"
-    taken.each do |record|
-      puts record.title
-    end
-    sleep 2
-  end
-  
-  taken.records.each do |final_taken|
-
-  #SCRIPTING and EDITING
-  if SCRIPTING
-    print $clear_code
-    print "\a"
-    say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
-    say("SCRIPTING...".bold.white.on_black)
-    final_taken.local_script(scripting) 
-  end
-  
-  if EDITING
-    print $clear_code
-    print "\a"
-    say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
-    edit = ask("Would you like to " + "edit".headline + " this record [yes or ANYTHING]? ")
-    if edit == ('y' or 'yes')
-      print $clear_code
-      say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
-      final_taken.edit
-    end
-  end
-  
-  print $clear_code
-  say("#{'-'*20}FINAL RECORD#{'-'*20}".headline)
-  say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
-  say("#{'-'*20}FINAL RECORD#{'-'*20}".headline)
-  to_save_or_not_to_save = ask("Would you like to keep this FINAL RECORD? (ANYTHING or n)")
-
-  if to_save_or_not_to_save == 'n' 
-    next #if continue? == 'again' 
-  end
-
-  if CSV
-    print $clear_code
-    print "\a"
-    say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
-    say("Label making time!".headline)
-    csv = final_taken.marc_to_csv(labels) 
-    print "Final csv:\n#{csv}\n"  
-  end
-  
-  print $clear_code
-  clear_code_do = false
-
-  
-  #saves in the directory in which zcc was run
-  if HOW_TO_SAVE == 'search_term'
-    filename = search_term.gsub(' ','_')
-  elsif HOW_TO_SAVE == 'timestamp'
-    filename = TIMESTAMP 
-  end
-  
-  FORMAT_TO_SAVE.each do |format|
-    if format == 'marc'
-      puts
-      aFile = File.new("#{filename}\.mrc","a+")
-      aFile.write(final_taken.marc.to_marc)
-      puts "#{final_taken.marc['245']['a']} saved in #{filename}\.mrc"
-      aFile.close
-      puts
-    elsif format == 'xml'
-      #puts final_taken.marc.to_xml
-      aFile = File.new("#{filename}\.xml","a+")      
-      aFile.write(final_taken.marc.to_xml)
-      puts "#{final_taken.marc['245']['a']} saved in #{filename}\.xml"
-      aFile.close
-    elsif format == 'zebra'
-      #puts final_taken.to_xml
-      File.open("#{File.expand_path("~")}/.zcc/zebra/records/#{Time.now.strftime("%Y%m%d%H%M%S")}\.mrc", "w") do |f|
-        f.write final_taken.marc.to_marc
-      end
-      Dir.chdir("#{File.expand_path("~")}/.zcc/zebra") do
-        record_update = `zebraidx update records`
-        puts record_update
-      end
-    #saving to koha2 is untested. Please let me know if it doesn't work--or even if it does.
-    elsif format == 'koha2'
-      puts
-      out = Tempfile.new("koha2-")
-      out << final_taken.marc.to_marc
-      out.close
-      koha_save = `perl /usr/local/koha/intranet/scripts/misc/migration_tools/bulkmarcimport.pl -v 6 -file #{out.path}`
-      puts koha_save
-      puts
-    end
-  end
-  
-    if CSV
-      puts
-      csvFile = File.new("csv-#{filename}\.txt","a+")
-      csvFile.write(csv)
-      puts "csv for #{final_taken.marc['245']['a']} saved in csv-#{filename}\.txt"
-      csvFile.close
-      puts
-    end
-  end
-  
-}
+#!/usr/bin/ruby -w
+## Jason Ronallo
+# July, 2007
+#zcc-new
+
+require 'rubygems'
+require 'zoom'
+require 'marc'
+require 'highline/import'
+require 'term/ansicolor'
+require 'getoptlong'
+#include Term::ANSIColor
+
+require 'stringio'
+require 'yaml'
+require 'tempfile'
+
+$KCODE = 'u'
+require 'jcode'
+require 'unicode'
+
+require 'zcc'
+include ZCC
+$testing = false
+opts = GetoptLong.new( [ '--yaml', '-y', GetoptLong::REQUIRED_ARGUMENT ],
+                       ['--testing', '-t', GetoptLong::NO_ARGUMENT]
+                      )
+opts_yaml =''
+opts.each do |opt, arg|
+  case opt
+    when '--yaml'
+      opts_yaml = arg
+    when '--testing'
+      $testing = true
+  end
+end
+
+if opts_yaml.length > 0
+  file_yaml = opts_yaml
+else
+  file_yaml = 'zcc.yaml'
+end
+
+termsize = HighLine::SystemExtensions.terminal_size
+$term_width = termsize[0]
+$term_height = termsize[1]
+$clear_code = %x{clear}
+HighLine.track_eof = false
+
+##Setting up testing parameter
+#$testing = ARGV[0]
+#$testing = nil unless $testing == 'v' or $testing == 't'
+
+## FIXME: make use of a 'file' version of zcc where records are not taken from zservers but from a file. Way to process a batch of records with the same scripting.
+file_mode = true if ARGV[0] == 'file'
+
+#reading in config from yaml file
+begin
+  yamlfile = YAML.load_file( "#{file_yaml}" )
+rescue Exception
+  puts "You must either start zcc in the same directory as a file named 'zcc.yaml' or give the '-yaml' switch with the full path to the yaml file which includes configuration information. This feature allows you to start zcc with different configuration files.\nzcc -yaml /home/jason/.zcc/zcc.yaml" 
+  exit
+end
+
+labels = yamlfile['labels']
+scripting = yamlfile['scripting']
+#puts scripting.inspect
+#gets
+$procs = yamlfile['procs']
+
+TO_SHOW = yamlfile['to_show']
+SHOW_PER_PAGE = yamlfile['show_per_page']
+FIELDS_TO_SHOW = yamlfile['fields_to_show']
+HOW_TO_SAVE = yamlfile['save_filename']
+TIMESTAMP = Time.now.strftime("%Y%m%d%H%M%S")
+SCRIPTING = yamlfile['scripting_on']
+EDITING = yamlfile['subfield_editing']
+FULL_EDITING = yamlfile['full_editing']
+TEXT_EDITOR = yamlfile['text_editor']
+CSV = yamlfile['csv_on']
+LINTER = yamlfile['linter_on']
+FORMAT_TO_SAVE = yamlfile['save_record_syntax']
+ROOT = yamlfile['root']
+START_ZEBRA = yamlfile['start_zebra']
+KILL_ZEBRA = yamlfile['kill_zebra']
+
+
+
+def get_user_pass
+  user_pass = IO.read("#{ROOT}/zebra/zcc_passw").chomp
+  user, pass = user_pass.split(":")  
+end
+USER, PASS = get_user_pass
+puts "user |" + USER + "|"
+puts "pass |" + PASS + "|"
+
+def quit_routine
+  #print "killing zebra: "
+  puts $zebra_pid if $zebra_pid
+  Process.kill('TERM', $zebra_pid) if KILL_ZEBRA 
+  exit
+end
+
+print $clear_code
+#printed once when starting the program.
+say("ZCC v. 0.2.0.\nZCC, Copyright (C) 2007 Jason Ronallo\nZcc comes with ABSOLUTELY NO WARRANTY.\nZcc is released under the terms of the GPL v.2 or later.\nSee LICENSE for full information.\n".bold)
+say("Simply enter a title, LCCN or ISBN and hit enter.\nTitle: enter the title exactly as it is on the source\n   for better relevancy ranking by title.\nLCCN: Include the dash, like 81-21962\nEnter ISBNs without dashes.\nNot all servers accept it, but you can also do a narrowing search\nby including an author's name after the title like so:\nFinnegans Wake :au Joyce")
+say("For more help see: http://zcc.rubyforge.org/zcc.html")
+
+#start zebra server if
+if START_ZEBRA
+  Dir.chdir("#{ROOT}/zebra") do
+    $zebra_pid = fork do
+      STDERR.close
+      exec "zebrasrv @:9998 -l zebrasrv.log"
+    end
+  end
+else
+  $zebra_pid = false
+end
+
+zservers = yamlfile['zservers']
+
+if zservers[0][0] =~ /Include/
+  filename = zservers[0][0][8, 99]
+  contents = YAML.load_file("#{ROOT}/#{filename}")
+  contents.each{|c| zservers.push(c)}
+  zservers[0] = nil
+end
+zservers.compact!
+
+zserver_objects = []
+zservers.each do |server|
+  zserver_objects << Zserver.new(server[0], server[1], server[2], server[3])
+end
+
+server_h = Hash.new {|hash, key| hash[key] = [] }
+zserver_objects.each {|z| server_h[z.group] << z}
+
+groups = []
+server_h.each_key{|k| groups << k}
+groups.uniq!
+groups.sort!
+
+loop {
+  search_term = ask("\nEnter search. [ISBN, title, LCCN (with dash), or :q to quit]\nsearch> ".boldz, String){ |q| q.readline = true }
+    quit_routine if search_term == ':q'
+    
+  taken = ResultSet.new  
+  groups.each do |group|
+    print $clear_code
+    servers = server_h[group]
+    query = Query.new(search_term, servers)
+    results = query.search(TO_SHOW)
+    results.index_pos = SHOW_PER_PAGE - 1
+    if results.records.empty?
+      puts "There are no results from group #{group} servers!\n\n\n"
+      another_group = ask("Do you want to continue searching the next group of servers? [y], anything  ".boldz)
+      if another_group == 'y' || another_group == ''
+        next
+      else
+        print $clear_code
+        break
+      end
+    end
+    results.rank_by_relevance!
+    return_value = zcc_select_good_marc(results, 'multi')
+    quit_routine if return_value == 'quit'
+    taken << results
+    break if return_value == 'done'    
+  end
+  
+      taken.remove_unselected!
+      next if taken.empty?
+      continue = true
+      print $clear_code
+      #say("\aThis is your combined result set.\nWe will winnow until you choose ".white.bold.on_black + "'done'".red.bold + "\nor you unselect all records.".white.bold.on_black)
+      #sleep 1
+      while continue
+       return_value = zcc_select_good_marc(taken, 'winnow')
+       quit_routine if return_value == 'quit'
+       continue = false if taken.selected_length == 0
+       continue = false if return_value == 'done'
+      end
+      taken.remove_unselected!
+  unless taken.length == 0
+    puts "Here are the records you've selected for processing:"
+    taken.each do |record|
+      puts record.title
+    end
+    #sleep 2
+  end
+  
+  taken.records.each do |final_taken|
+    #puts final_taken.class; STDIN.gets
+  #SCRIPTING and EDITING
+  def which_script scripting
+    loop do
+      choose do |scripting_menu|
+        scripting_menu.layout = :list
+        scripting_menu.index = :number
+        scripting_menu.readline = true
+        scripting_menu.shell  = true
+        scripting_menu.prompt = "Enter which script you would like to run on this record: "
+        scripting_menu.select_by = :index_or_name
+        scripting_menu.choice(:none, "No script will be run.") {|cmd, d| return 'none'}
+        scripting.each do |key, value|
+          unless key == 'start' || key == 'end'
+            scripting_menu.choice(key, "no help, ha!") do |cmd, d| 
+              if cmd == :help
+                next
+              else
+                return scripting[cmd]
+              end
+            end
+          end
+        end
+      end        
+    end
+  end
+
+  def display_record record
+    print $clear_code
+    print "\a"
+    say("#{ZCC.zcc_marc_str_bold(record.to_s, 'record')}")
+  end
+  
+  if SCRIPTING
+    display_record(final_taken)
+    picked_scripting = which_script scripting    
+    display_record(final_taken)
+    say("SCRIPTING...".bold.white.on_black)
+    final_taken.local_script(scripting["start"]) if scripting["start"] && picked_scripting != 'none'
+    final_taken.local_script(picked_scripting) unless picked_scripting == 'none'
+    final_taken.local_script(scripting["end"]) if scripting["end"] && picked_scripting != 'none'
+  end
+  
+  if EDITING
+    display_record(final_taken)
+    edit = ask("Would you like to " + "edit any subfields".headline + " in this record [yes or ANYTHING]? ")
+    if edit == ('y' or 'yes')
+      print $clear_code
+      say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
+      puts final_taken.class
+      final_taken.edit
+    end
+  end
+
+  
+  if FULL_EDITING
+    display_record(final_taken)
+    edit2 = ask("Would you like to " + "edit".headline + " this record in" + " #{TEXT_EDITOR} ".headline + "[yes or ANYTHING]? ")
+     if edit2 == ('y' or 'yes')
+      print $clear_code
+      say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
+      final_taken.full_edit
+    end
+  end
+  
+  print $clear_code
+  say("#{'-'*20}FINAL RECORD#{'-'*20}".headline)
+  say("#{ZCC.zcc_marc_str_bold(final_taken.to_s, 'record')}")
+  say("#{'-'*20}FINAL RECORD#{'-'*20}".headline)
+  to_save_or_not_to_save = ask("Would you like to keep this FINAL RECORD? (ANYTHING or n)")
+
+  if to_save_or_not_to_save == 'n' 
+    next #if continue? == 'again' 
+  end
+
+  if CSV
+    display_record(final_taken)
+    say("Label making time!".headline)
+    csv = final_taken.marc_to_csv(labels) 
+    print "Final csv:\n#{csv}\n"  
+  end
+  
+  print $clear_code
+  clear_code_do = false
+
+  
+  #saves in the directory in which zcc was run
+  if HOW_TO_SAVE == 'search_term'
+    filename = search_term.gsub(' ','_')
+  elsif HOW_TO_SAVE == 'timestamp'
+    filename = TIMESTAMP 
+  end
+  
+  FORMAT_TO_SAVE.each do |format|
+    if format == 'marc'
+      puts
+      aFile = File.new("#{filename}\.mrc","a+")
+      aFile.write(final_taken.marc.to_marc)
+      puts "#{final_taken.marc['245']['a']} saved in #{filename}\.mrc"
+      aFile.close
+      puts
+    elsif format == 'xml'
+      #puts final_taken.marc.to_xml
+      aFile = File.new("#{filename}\.xml","a+")      
+      aFile.write(final_taken.marc.to_xml)
+      puts "#{final_taken.marc['245']['a']} saved in #{filename}\.xml"
+      aFile.close
+    elsif format == 'zebra'
+      #create the recordID which zebra will use to index
+      t = Time.now
+      the_time = "%10.6f" % t.to_f
+      final_taken.marc.append(MARC::DataField.new('901', '0', '1', ['a', "#{the_time}"])) unless final_taken.marc['901'] && final_taken.marc['901']['a']
+
+      #connect to zebrasrv and insert record
+      conn = ZOOM::Connection.new('user' => USER, 'password' => PASS).connect('localhost:9998/zcc')
+      p = conn.package
+      p.wait_action = 'waitIfPossible'
+      p.action = 'specialUpdate'
+      p.record = final_taken.marc.to_xml
+      p.send('update')
+      p.send('commit')
+      puts "Finished commit of #{final_taken.marc['245']['a']} to zebra server."
+
+      #place backup filesystem copy in zebra directory
+      #with extended services the record is saved to zebra internal store
+      #this also saves it to the file system
+      File.open("#{ROOT}/zebra/records/#{final_taken.marc['901']['a']}\.xml", "w") do |f|
+        f.write final_taken.marc.to_xml
+      end
+    elsif format == 'koha2'
+      puts
+      out = Tempfile.new("koha2-")
+      out << final_taken.marc.to_marc
+      out.close
+      koha_save = `perl /usr/local/koha/intranet/scripts/misc/migration_tools/bulkmarcimport.pl -v 6 -file #{out.path}`
+      puts koha_save
+      puts
+    end
+  end
+  
+    if CSV
+      puts
+      csvFile = File.new("csv-#{filename}\.txt","a+")
+      csvFile.write(csv)
+      puts "csv for #{final_taken.marc['245']['a']} saved in csv-#{filename}\.txt"
+      csvFile.close
+      puts
+    end
+  end
+  
+}
+
+
+ 
+
+
+
+