marc2solr 0.1.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 BillDueber
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# marc2solr -- get MARC data into Solr
+
+`marc2Solr` is a package wrapping up functionality in a variety of other gems, designed to make getting data from [MARC21](http://en.wikipedia.org/wiki/MARC_standards) files into [Solr](http://lucene.apache.org/Solr/) as painless as possible.
+
+`marc2Solr` is based on [Solrmarc](http://code.google.com/p/solrmarc/), the excellent Java-based program that does more or less the same thing. `marc2Solr` is *not* a drop-in replacement for Solrmarc, but can do most of the same things. A naive program to translate solrmarc config files to marc2solr config files is included. It's called -- wait for it -- solrmarc_to_marc2solr.
+
+It relies on [jruby](http://jruby.org/) to pull it all together; this will not run under stock Ruby!
+
+## Documentation
+
+* [The marc2solr wiki]() has documentation on how to install, configure, and use `marc2solr`, how it compares to `solrmarc`, etc.
+* The [marc2solr_example](http://github.com/billdueber/marc2solr_example) git project has two examples: `simple_sample` has a very simple index and some translation maps that show off the major features with plenty of documentation. The `umich` subdirectory is the actual working code for the University of Michigan [mirlyn](http://mirlyn.lib.umich.edu/) install.
+* The [marcspec wiki](http://github.com/billdueber/marcspec/wiki/) is the definitive source for how to construct your index file, translation maps, and custom functions.
+
+
+
+## Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+## Copyright
+
+Copyright (c) 2010 BillDueber. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "marc2solr"
+    gem.summary = %Q{MARC2Solr: Get MARC into Solr via JRuby}
+    gem.description = %Q{Given a file of MARC records, send them to Sorl for indexing based on a set of MARCSpecs}
+    gem.email = "bill@dueber.com"
+    gem.homepage = "http://github.com/billdueber/marc2solr"
+    gem.authors = ["BillDueber"]
+    
+    gem.add_dependency 'marc4j4r', '>= 1.2.0'
+    gem.add_dependency 'jruby_streaming_update_solr_server', '>=0.5.2'
+    gem.add_dependency 'marcspec', '>= 1.6.3'
+    gem.add_dependency 'threach', '>= 0.2.0'
+    gem.add_dependency 'logback-simple'
+
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "yard", ">= 0"
+
+    gem.bindir = 'bin'
+    gem.executables << 'solrmarc_to_marc2solr'
+    gem.executables << 'marc2solr'
+    
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/bin/marc2solr

@@ -0,0 +1,247 @@
+#!/usr/bin/env jruby --server -J-Djruby.compile.frameless=true -J-Djruby.compile.positionless=true -J-Djruby.compile.peephole=true
+require 'marc2solr'
+require 'marc2solr/marc2solr_custom'
+
+require 'rubygems'
+require 'pp'
+require 'logback-simple'
+require 'marcspec'
+require 'threach'
+
+
+
+
+opts = MARC2Solr::Conf.new
+
+# pp opts
+
+
+###########################
+# Get a master logger
+###########################
+
+
+$LOG = opts.masterLogger
+
+# Perform the command
+
+case opts.command
+  
+#################
+# Commit -- just send a commit to the configured solr
+#################
+  
+when "commit"
+  $LOG.info "Commit to #{opts.sussURL}"
+  if opts[:dryrun]
+    $LOG.debug "Using javabin" if opts[:javabin]
+    $LOG.info "DRY RUN. Stopping now."
+  else
+    opts.suss.commit
+    $LOG.info "Commit done"
+  end
+  
+
+#####################################################  
+# delete -- delete IDs listed in the given file(s)
+#####################################################  
+
+when "delete"
+  delfiles = opts.rest
+  unless delfiles.size > 0
+    $LOG.error "command 'delete' needs at least one filename"
+    puts "\n\nERROR: command 'delete' needs at least one filename"
+    opts.print_command_help('delete')
+  end
+  
+  # Get the suss
+  suss = opts.suss
+  
+  # Make sure they can all be opened
+  delfiles.each do |filename|
+    unless File.readable? filename
+      $LOG.error "Can't open configuration file `#{filename}`"
+      raise ArgumentError, "Can't open configuration file `#{filename}`" 
+    end
+  end
+  
+  # Now go ahead and process them
+  total = 0
+  
+  delfiles.each do |filename|
+    count = 0
+    f = File.open(filename)
+    $LOG.info "Deleting IDs listed in #{filename}"
+    $LOG.info "DRY RUN ONLY" if opts[:dryrun]
+    f.each_line do |id|
+      id.chomp!
+      suss.deleteById(id) unless opts[:dryrun]
+      $LOG.debug "Deleted id\t#{id}"
+      count += 1
+      total += 1
+    end
+    f.close
+    $LOG.info "Tried to delete #{count} ids from file '#{filename}'"
+  end
+  $LOG.info "Tried to delete #{total} ids from all #{delfiles.size} files" if delfiles.size > 1
+
+  unless opts[:dryrun] or opts[:skipcommit]
+    $LOG.info "Sending final commit"
+    suss.commit
+    $LOG.info "Final commit finished"
+  end
+
+
+#####################################################  
+# index -- index the given marc files
+#####################################################  
+
+when "index"
+  marcfiles = opts.rest
+  
+  unless marcfiles.size > 0
+    $LOG.error "command 'delete' needs at least one marc file to index"
+    puts "\n\nERROR: command 'index' needs at least one filename"
+    opts.print_command_help('index')
+  end
+  
+  
+  # Make sure everything can be opened
+  unless File.readable? opts[:indexfile]
+    $LOG.error "Index configuration file '#{opts[:indexfile]}' (set with --indexfile) cannot be found/read"
+    raise ArgumentError, "Index configuration file '#{opts[:indexfile]}' (set with --indexfile) cannot be found/read"
+  else
+    $LOG.debug "Found index file #{opts[:indexfile]}"
+  end
+  
+  unless File.readable? opts[:tmapdir]
+    $LOG.error "Translation Map directory '#{opts[:tmapdir]}' (set with --tmapdir) cannot be found/read"
+    raise ArgumentError, "Translation Map directory '#{opts[:tmapdir]}' (set with --tmapdir) cannot be found/read"
+  else
+    $LOG.debug "Found translation maps directory #{opts[:tmapdir]}"
+  end
+  
+  marcfiles.each do |filename|
+    if filename == "STDIN"
+      $LOG.info "Using standard input as a marc file"
+      next
+    end
+    unless File.readable? filename
+      $LOG.error "Can't open MARC file `#{filename}`"
+      raise ArgumentError, "Can't open MARC file `#{filename}`" 
+    else
+      $LOG.debug "Adding marc file #{filename} to queue"
+    end
+  end
+  
+
+  # Load all the files in the customdir(s)
+  $LOG.debug "Loading custom routines"
+  if opts[:customdir].size > 0
+    opts[:customdir].each do |dir|
+      unless File.exist? dir
+        $LOG.warn "Skipping load directory '#{dir}': Not found"
+        opts[:customdir].delete dir
+      end
+      $LOG.info "Loading files in #{dir}"
+      Dir.glob(["#{dir}/*.jar"]).each do |x|
+        $LOG.debug "Loading #{x}"
+        require x
+      end
+      Dir.glob(["#{dir}/*.rb"]).each do |x|
+        $LOG.debug "Loading #{x}"
+        require x
+      end
+    end
+  end
+
+  # Get the suss. Will throw an Argument error if there's not enough information
+  suss = opts.suss
+    
+  # Create a specset
+
+  ss = MARCSpec::SpecSet.new
+  ss.loadMapsFromDir opts[:tmapdir]
+  ss.buildSpecsFromDSLFile opts[:indexfile]
+  
+  # Set up if we're using threach or not
+  if opts[:threads] > 1
+    method = :threach
+    args = [opts[:threads], :each_with_index]
+  else
+    method = :each_with_index
+    args = []
+  end
+  
+  
+  if opts[:dryrun]
+    $LOG.info "Begin DRY RUN; nothing will be sent to Solr"
+  end
+  
+  # Clear solr if so declared
+  if opts[:clearsolr] 
+    if opts[:dryrun]
+      $LOG.info "Would have cleared out solr (but for dryrun)"
+    else
+      suss.deleteByQuery('*:*')
+      suss.commit
+      $LOG.info "Cleared out solr"
+    end
+  end
+  
+  # Read each filename in turn, indexing records in each 
+
+  i = 0 # Declare out here so we have a counter independent of the file being indexed
+  
+  start = Time.new.to_f
+  marcfiles.each do |filename|
+    $LOG.info "Indexing file #{filename}"
+    
+    reader = opts.reader(filename)
+    reader.send(method, *args) do |r, i|
+      Thread.current[:index] = i
+      
+      $LOG.debug("Indexing record with 001 {}", r['001'])
+      
+      doc = ss.doc_from_marc(r, opts[:benchmark])
+      
+      # Send it to solr
+      unless opts[:dryrun]
+        suss << doc
+      end
+      
+      # Print out the record and/or the document
+      opts[:debugfile].puts r   if opts[:printmarc]
+      opts[:debugfile].puts doc if opts[:printdoc]
+      opts[:debugfile].puts "\n\n" if opts[:printdoc] or opts[:printmarc]
+      
+      if Thread.current[:index] % opts[:logbatchsize] == 0
+        pace  = Thread.current[:index] / (Time.new.to_f - start) 
+        $LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [Thread.current[:index], pace]
+      end
+    end # reader
+  end # marcfiles
+  
+  # Commit
+  
+  unless opts[:dryrun] or opts[:skipcommit]
+    $LOG.info "Sending final commit"
+    suss.commit
+    $LOG.info "Final commit finished"
+  end
+  
+  # Be done
+  $LOG.info "Done indexing"
+  pace  = i / (Time.new.to_f - start) 
+  $LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [i, pace]
+  
+  
+  # Log the benchmarking information if requested
+  if opts[:benchmark]
+    ss.benchmarks.keys.sort{|a,b| ss.benchmarks[b].real <=> ss.benchmarks[a].real}.each do |k|
+      $LOG.info("%-20s %s" % [k + ':', ss.benchmarks[k].real.to_s])
+    end
+  end
+  
+end # end of the case statement
+

data/bin/solrmarc_to_marc2solr

@@ -0,0 +1,260 @@
+#!/usr/bin/env jruby
+require 'marc2solr/marc2solr_custom'
+require 'rubygems'
+require 'marc4j4r'
+require 'logback-simple'
+require 'pp'
+require 'marcspec'
+require 'fileutils'
+
+
+# We'll take two arguments: a .properties index file, and a new directory 
+propfile = ARGV[0]
+newdir = ARGV[1]
+
+def usage
+  puts %q{
+  solrmarc_to_marc2solr -- attempt to convert a solrmarc index file and
+  translation maps to marc2solr format
+  
+  Usage:
+  
+    solrmarc_to_marc2solr /path/to/blah_index.properties /path/to/newdir
+    
+  This will:
+   1. Tranlaste all the translation maps found in the directory "translation_maps"
+      located in the same direcotry as blah_index.properites and put them in
+      /path/to/newdir/translation_maps
+   2. Translate blah_index.properties and put the resultling index.dsl file in
+      /path/to/newdir/index.dsl
+   3. Create a logfile, solrmarc_to_marc2solr.log in /path/to/newdir/ that
+      lists the lines it couldn't read and didn't attempt to translate (e.g., most
+      custom functions)
+  }
+  Process.exit
+  
+end
+
+# Need help?
+
+if ARGV.size != 2
+  usage
+end
+
+
+$LOG = Logback::Simple::Logger.singleton('fromsolrmarc')
+Logback::Simple.loglevel = :debug
+Logback::Simple.startFileLogger("#{newdir}/fromsolrmarc.log")
+
+
+# First, try to create the new directory structure
+
+
+
+unless File.exist? propfile
+  STDERR.puts  "Can't find index file '#{propfile}\nUse 'solrmarc_to_marc2solr' by itself for usage"
+  $LOG.error "Can't find file '#{propfile}"
+  Process.exit
+end
+
+unless File.readable? propfile
+  STDERR.puts "Index file '#{propfile}' exists but cannot be read\nUse 'solrmarc_to_marc2solr' by itself for usage"
+  $LOG.error "File '#{propfile}' exists but cannot be read"
+  Process.exit
+end
+
+
+begin
+  FileUtils.mkdir_p "#{newdir}/translation_maps"
+  FileUtils.mkdir "#{newdir}/lib"
+rescue Exception => e
+  $LOG.debug e
+end
+
+
+
+
+propfiledir = File.dirname(propfile)
+trmapdir = propfiledir + '/translation_maps'
+newpropfile = 'index.dsl'
+
+
+
+
+
+ss  = MARCSpec::SpecSet.new
+Dir.glob(trmapdir + '/*.properties').each do |f|
+  File.open(f) do |fh|
+    fh.each_line do |line|
+      next if line =~ /^\s*#/
+      next unless line =~ /\S/
+      if line =~ /^\s*pattern/
+        $LOG.debug "Adding '#{File.basename f}' as a pattern map"
+        ss.add_map MARCSpec::MultiValueMap.from_solrmarc_file(f)
+        break
+      else
+        $LOG.debug "Adding '#{File.basename f}' as a key/value map"
+        ss.add_map MARCSpec::KVMap.from_solrmarc_file(f)
+        break
+      end
+    end
+  end
+end
+
+
+
+WHOLE = /^(\d{3})$/
+CTRL = /^(\d{3})\[(.+?)\]/
+VAR  = /^(\d{3})(.+)/
+
+File.open(propfile) do |fh|
+  fh.each_line do |line|
+    next unless line =~ /\S/
+    line.strip!
+    
+    # Leave comments alone
+    if line =~ /^#/
+      # puts line
+      next
+    end
+    
+    fieldname,spec = line.split(/\s*=\s*/)
+    
+    # Deal with constants
+    if spec =~ /^"(.+)"\s*$/
+      constant = $1
+      csf = MARCSpec::ConstantSolrSpec.new(:solrField=>fieldname, :constantValue=>constant)
+      ss << csf
+      next
+    end
+    
+    # Deal with built-in functions if we can
+    if spec == 'FullRecordAsXML'
+      csf = MARCSpec::CustomSolrSpec.new(:solrField=>fieldname,
+                                          :module => MARC2Solr::Custom,
+                                          :functionSymbol => :asXML)
+      ss << csf
+      next
+    end
+
+    if spec == 'FullRecordAsMARC'
+      csf = MARCSpec::CustomSolrSpec.new(:solrField=>fieldname,
+                                          :module => MARC2Solr::Custom,
+                                          :functionSymbol => :asMARC)
+      ss << csf
+      next
+    end
+    
+    if spec == 'DateOfPublication'
+      csf = MARCSpec::CustomSolrSpec.new(:solrField=>fieldname,
+                                          :module => MARC2Solr::Custom,
+                                          :functionSymbol => :getDate)
+      ss << csf
+      next
+    end
+      
+    
+    if spec =~ /^custom,\s*getAllSearchableFields\((\d+),\s*(\d+)\)/
+      low = $1
+      high = $2
+      csf = MARCSpec::CustomSolrSpec.new(:solrField=>fieldname,
+                                         :module => MARC2Solr::Custom,
+                                         :functionSymbol => :getAllSearchableFields,
+                                         :functionArgs => [low, high])
+      ss << csf
+      next
+    end
+    
+    # Log and ignore other custom fields
+    if spec =~ /^custom/
+      $LOG.warn "Skipping custom line #{line}"
+      next
+    end
+    
+      
+      
+    #otherwise, build one from scratch
+    
+    sfs = MARCSpec::SolrFieldSpec.new(:solrField => fieldname)
+    
+    marcfields, *specials = spec.split(/\s*,\s*/)
+    
+    marcfields.split(/\s*:\s*/).each do |ms|
+      if WHOLE.match ms
+        tag = $1
+        if MARC4J4R::ControlField.control_tag? tag
+          sfs << MARCSpec::ControlFieldSpec.new(tag)
+        else
+          sfs << MARCSpec::VariableFieldSpec.new(tag)
+        end
+        next
+
+      elsif CTRL.match ms
+        tag = $1
+        range = $2
+        first,last = range.split('-')
+        last ||= first
+        first = first.to_i
+        last = last.to_i
+        sfs << MARCSpec::ControlFieldSpec.new(tag, first..last)
+        next
+      elsif VAR.match ms
+        tag = $1
+        sfcodes = $2.split(//)
+        sfs << MARCSpec::VariableFieldSpec.new(tag, sfcodes)
+      else
+        $LOG.warn "Didn't recognize line '#{line}'"
+      end
+    end # marcfields.split
+    
+    # Add in the specials -- "first", maps, etc.
+    specials.each do |special|
+      case special
+      when 'first'
+        sfs.first = true
+      else
+        origmapname = special
+        mapname =  special.gsub(/.properties/, '')
+        sfs.map = ss.map(mapname)
+        if mapname.nil? 
+          $LOG.warn "Map problem in #{fieldname}: Unrecognized map name '#{mapname}' (specified as '#{origmapname}')"
+        end
+        if mapname =~ /\((.*)\)/
+          $LOG.warn "Map problem in #{fieldname}: Translator doesn't deal at all with property key prefixes ('#{$1}' in this case). Please break them into separate map files or complain to Bill."
+        end
+      end
+    end
+    ss << sfs if sfs.marcfieldspecs.size > 0
+  end
+end
+
+# Spit it out
+
+# First, put the maps in newdir/translation_maps
+
+ss.tmaps.each do |name, map|
+  filename = name + '.rb'
+  $LOG.debug "Writing out translation map #{filename}"
+  File.open("#{newdir}/translation_maps/#{filename}", 'w') do |f|
+    f.puts map.asPPString
+  end
+end
+
+# Now the solrspecs
+# File.open("#{newdir}/#{newpropfile}", 'w') do |f|
+#   $LOG.debug "Writing out spec file #{newpropfile}"
+#   f.puts '['
+#   ss.solrfieldspecs.each do |sfs|
+#     f.puts sfs.asPPString + ','
+#   end
+#   f.puts ']'
+# end
+
+File.open("#{newdir}/#{newpropfile}", 'w') do |f|
+  $LOG.debug "Writing out spec file #{newpropfile}"
+  ss.solrfieldspecs.each do |sfs|
+    f.puts sfs.asDSLString
+    f.puts ''
+  end
+end
+