marc2solr 0.1.1

data/lib/marc2solr/marc2solr_custom.rb

@@ -0,0 +1,194 @@
+require 'rubygems'
+require 'logback-simple'
+
+module MARC2Solr
+  module Custom
+    LOG = Logback::Simple::Logger.singleton
+    
+    # Custom routines are defined as module methods that take two arguments: a MARC4J4R record,
+    # and an (optional) array of other arguments passed in. 
+    #
+    # They don't need to live in the MARC2Solr::Custom namespace, but it's not a bad idea to use, e.g.,
+    # MARC2Solr::Custom::UMich, or maybe MARC2Solr::Custom::DateStuff
+    #
+    # You can return multiple values in an array
+    
+    # The simplest possible example; just call a method on the underlying MARC4J4R record
+    # Note that even though we don't use the arguments, the method signature has to 
+    # support it
+    #
+    # @param [hashlike] doc The document object being added to; allows you to leverage already-done work
+    # @param [MARC4J4R::Record] r A MARC4J4R record
+    # @param [#[]] doc A hashlike (responds to #[]) that holds the computed values for fields "so far"
+    # @return [String] The XML representation of the record
+
+    def self.asXML doc, r  #Remember, module fucntion! Define with "def self.methodName"
+      return r.to_xml
+    end
+    
+    # Another for marc binary
+    def self.asMARC doc, r
+      return r.to_marc
+    end
+    
+    
+    # And another for marc-in-json
+    
+    def self.as_marc_in_json doc, r
+      return r.to_marc_in_json
+    end
+    
+    # Here we get all the text from fields between (inclusive) the two tag strings in args;
+    # 
+    # @param [hashlike] doc The document object being added to; allows you to leverage already-done work
+    # @param [MARC4J4R::Record] r A MARC4J4R record
+    # @param [Array<String>] args An array of two strings, the lowest tag you want to include, and
+    # the highest
+    # @return [String] A single single string with all the text from included fields
+    def self.getAllSearchableFields(doc, r, lower, upper)
+      data = []
+      r.each do |field|
+        next unless field.tag <= upper and field.tag >= lower
+        data << field.value
+      end
+      return data.join(' ')
+    end
+    
+    # How about one to sort out, say, the 035s? We'll make a generic routine
+    # that looks for specified values in specified subfields of variable
+    # fields, and then make sure they match before returning them.
+    #
+    # See the use of this in the simple_sample/simple_index.rb file for field 'oclc'
+    #
+    # @param [hashlike] doc The document object being added to; allows you to leverage already-done work
+    # @param [MARC4J4R::Record] r A MARC4J4R record
+    # @param [String] tag A tag string (e.g., '035')
+    # @param [String, Array<String>] codes A subfield code ('a') or array of them (['a', 'c'])
+    # @param [Regexp] pattern A pattern that must match for the value to be included
+    # @param [Fixnum] matchindex The number of the substring captured by parens in the pattern to return
+    # The default is zero, which means "the whole string"
+    # @return [Array<String>] a (possibly empty) array of found values
+    def self.valsByPattern(doc, r, tag, codes, pattern, matchindex=0)
+      data = []
+      r.find_by_tag(tag).each do |f|
+        f.sub_values(codes).each do |v|
+          if m = pattern.match(v)
+            data << m[matchindex]
+          end
+        end
+      end
+      data.uniq!
+      return data
+    end
+    
+    
+    # An example of a DateOfPublication implementation
+    # @param [hashlike] doc The document object being added to; allows you to leverage already-done work
+    # @param [MARC4J4R::Record] r A MARC4J4R record
+    # @return [String] the found date, or nil if not found.
+    
+    def self.getDate doc, r
+      begin
+        ohoh8 = r['008'].value
+        date1 = ohoh8[7..10].downcase
+        datetype = ohoh8[6..6]
+        if ['n','u','b'].include? datetype
+          date1 = ""
+        else 
+          date1 = date1.gsub('u', '0').gsub('|', ' ')
+          date1 = '' if date1 == '0000'
+        end
+
+        if m = /^\d\d\d\d$/.match(date1)
+          return m[0]
+        end
+      rescue
+       # do nothing ... go on to the 260c
+      end
+
+
+      # No good? Fall back on the 260c
+      begin
+        d =  r['260']['c']
+        if m = /\d\d\d\d/.match(d)
+          return m[0]
+        end
+      rescue
+        LOG.debug "Record #{r['001']} has no valid date"
+        return nil
+      end
+    end    
+    
+    # A simple function to pull the non-indexing characters off the front of a field
+    # based on the second indicator
+    def self.fieldWithoutIndexingChars doc, r, tag
+      vals = []
+      r.find_by_tag(tag).each do |df|
+        ind2 = df.ind2.to_i
+        if ind2 > 0
+          vals << df.value[ind2..-1]
+        end
+      end
+      return vals
+    end
+    
+    
+    # A helper function -- take in a year, and return a date category
+    def self.getDateRange(date, r)
+      if date < "1500"
+        return "Pre-1500"
+      end
+
+      case date.to_i
+      when 1500..1800 then 
+        century = date[0..1]
+        return century + '00' + century + '99'
+      when 1801..2100 then
+        decade = date[0..2]
+        return decade + "0-" + decade + "9";
+      else
+  #      puts "getDateRange: #{r['001'].value} invalid date #{date}"
+      end
+    end
+    
+    
+    # Get the date range, based on the previously-computed pubdate
+      def self.pubDateRange(doc, r, wherePubdateIsStored)
+       previouslyComputedPubdate = doc[wherePubdateIsStored][0]
+       return [self.getDateRange(previouslyComputedPubdate)]
+      end
+    
+    
+    # We can do the same thing as a multi-return function -- compute the pubdate and
+    # the pubdaterange in one fell swoop. 
+    #
+    # In this case, we *could* just use the above self.pubDateRange. However, there
+    # are times when you several fields are based on intermediate values that you
+    # don't want to actually store in the solr document itself (e.g., a set of call number
+    # that you want to normalize or translate in a few different ways, without actually wanting
+    # to store the raw callnumbers in their own field). You may also need access to more metadata
+    # as you're constructing the data (e.g., you may want to store titles and titles-without-non-filing-
+    # character in different fields, but you can't compute one from the other wihout access to the
+    # associated indicator-2 value). 
+    #
+    # So, in this case, we'll get the pubDate and the pubDateRange all at once, just as an example,
+    # and put in the custom spec as:
+    #
+    # {
+    #   :solrField => ['pubDate', 'pubDateRange'],
+    #   :module => MARC2Solr::Custom,
+    #   :functionSymbol => :pubDateAndRange
+    # }
+    
+    
+    def self.pubDateAndRange(doc, r)
+      date = self.getDate(doc, r)
+      return [nil, nil] unless date
+      range = self.getDateRange(date, r)
+      return [date, range]
+    end
+    
+    
+  end # close the inner module Custom
+end # close the module MARC2Solr
+    

data/lib/marc2solr.rb

@@ -0,0 +1,452 @@
+require 'rubygems'
+
+require 'logback-simple'
+require 'trollop'
+require 'ftools'
+require 'jruby_streaming_update_solr_server'
+require 'marc4j4r'
+
+module MARC2Solr
+  
+  class Conf
+    include Logback::Simple
+    
+    SUB_COMMANDS = %w(index delete commit help)
+    
+    
+    OPTIONSCONFIG = [
+      [:config,    {:desc => "Configuation file specifying options. Repeatable. Command-line arguments always override the config file(s)",
+                    :type => :io,
+                    :multi => true}],
+      [:benchmark, {:desc=> "Benchmark production of each solr field",
+                    :only=> [:index],
+                    :short => '-B'
+                   }],
+     [:NObenchmark, {:desc=> "Benchmark production of each solr field",
+                   :only=> [:index],
+                  }],
+     [:dryrun,   {:desc => "Don't send anything to solr",
+                    }],
+      [:NOdryrun,   {:desc => "Disable a previous 'dryrun' directive",
+                    }],
+                    
+      [:printmarc, {:desc =>"Print MARC Record (as text) to --debugfile",
+                    :only => [:index],
+                    :short => '-r'
+                    }],
+      [:NOprintmarc, {:desc =>"Turn off printing MARC Record (as text) to --debugfile",
+                    :only => [:index],
+                    }],
+      [:printdoc,  {:desc => "Print each completed document to --debugfile", 
+                    :only => [:index],
+                    :short => '-d'}
+      ],
+      [:NOprintdoc,  {:desc => "Turn off printing each completed document to --debugfile", 
+                    :only => [:index],
+                    }],      
+      [:debugfile, {:desc => "Where to send output from --printmarc and --printdoc (takes filename, 'STDERR', 'STDOUT', or 'NONE') (repeatable)", \
+                    :default => "STDOUT",
+                    :isOutfile => true,
+                    :takesNone => true,
+                    :type => String, 
+                    :only => [:delete, :index],
+                    }],
+      [:clearsolr, {:desc => "Clean out Solr by deleting everything in it (DANGEROUS)",
+                    :only => [:index]
+                    }],
+      [:NOclearsolr, {:desc => "Disable a previous --clearsolr command",
+                  :only => [:index]
+                  }],
+      [:skipcommit,    {:desc => "DON'T send solr a 'commit' afterwards", 
+                    :short => '-C',
+                    :only => [:delete, :index],
+                    }],
+      [:threads,   {:desc => "Number of threads to use to process MARC records (>1 => use 'threach')", 
+                    :type => :int,
+                    :default => 1,
+                    :only => [:index]
+                    }],                    
+      [:sussthreads, {:desc => "Number of threads to send completed docs to Solr", 
+                      :type => :int,
+                      :default => 1}],
+      [:susssize,    {:desc => "Size of the documente queue for sending to Solr", 
+                      :short => '-S',
+                      :default => 128}],
+      [:machine, {:desc => "Name of solr machine (e.g., solr.myplace.org)",
+                      :short => '-m',
+                      # :required => [:index, :commit, :delete],
+                      :type => String}],
+      [:port,        {:desc => "Port of solr machine (e.g., '8088')", 
+                    :short => '-p',
+                    :type => :int}],
+      [:solrpath,  {:desc => "URL path to solr",
+                    :short => '-P',
+                   }],
+      [:javabin, {:desc => "Use javabin (presumes /update/bin is configured in schema.xml)", 
+                      }],                      
+      [:NOjavabin, {:desc => "Don't use javabin", 
+                      }],                      
+      [:logfile,   {:desc => "Name of the logfile (filename, 'STDERR', 'DEFAULT', or 'NONE'). 'DEFAULT' is a file based on input file name", 
+                    :default => "DEFAULT",
+                    :takesNone => true,                    
+                    :type => String}],
+      [:loglevel, {:desc=>"Level at which to log (DEBUG, INFO, WARN, ERROR, OFF)",
+                   :short => '-L',
+                   :takesNone => true,
+                   :valid => %w{OFF DEBUG INFO WARN ERROR },
+                   :default => 'INFO'}],
+      [:logbatchsize, {:desc => "Write progress information to logfile after every N records",
+                       :default => 25000,
+                       :only => [:delete, :index],
+                       :short => '-b'}],
+      [:indexfile, {:desc => "The index file describing your specset (usually index.dsl)",
+                    :type => String,
+                    :only => [:index],
+                    }],
+      [:tmapdir,   {:desc => "Directory that contains any translation maps",
+                    :type => String,
+                    :only => [:index]
+                    }],
+      [:customdir, {:desc=>"The directory containging custom routine libraries (usually the 'lib' next to index.rb). Repeatable",
+                    :only => [:index],
+                    :multi => true,
+                    :takesNone => true,
+                    :type => String
+                    }],
+      [:marctype, {:desc => "Type of marc file ('bestguess', 'strictmarc'. 'marcxml', 'alephsequential', 'permissivemarc')",
+                   :only => [:index],
+                   :short => '-t',
+                   :valid => %w{bestguess strictmarc permissivemarc marcxml alephsequential },
+                   :default => 'bestguess'
+                   }],      
+      [:encoding, {:desc => "Encoding of the MARC file ('bestguess', 'utf8', 'marc8', 'iso')",
+                   :valid => %w{bestguess utf8 marc8 iso},
+                   :only => [:index],
+                   :default => 'bestguess'}],
+      [:gzipped, {:desc=>"Is the input gzipped? An extenstion of .gz will always force this to true",
+                  :default => false,
+                  :only => [:index, :delete],
+                  }]
+
+    ]
+    
+    VALIDOPTIONS = {}
+    OPTIONSCONFIG.each {|a| VALIDOPTIONS[a[0]] = a[1]}
+    
+    
+    HELPTEXT = {
+      'help'  => "Get help on a command\nmarc2solr help <cmd> where <cmd> is index, delete, or commit",
+      'index' => "Index the given MARC file\nmarc2solr index --config <file> --override <marcfile> <marcfile2...>",
+      'delete' => "Delete based on ID\nmarc2solr delete --config <file> --override <file_of_ids_to_delete> <another_file...>",
+      'commit' => "Send a commit to the specified Solr\nmarc2solr commit --config <file> --override",
+    }
+    
+    attr_accessor :config, :cmdline, :rest, :command
+    def initialize
+      @config = {}
+      @cmdline = command_line_opts
+      
+      # Load the config files
+      if @cmdline[:config]
+        @cmdline[:config].each do |f|
+          log.info "Reading config-file '#{f.path}'"
+          self.instance_eval(f.read)
+        end
+      end
+      
+      # Remove the config
+      # Now override with the command line
+      @cmdline.delete :config
+      @cmdline.delete :config_given
+      
+      # Remove any "help" stuff 
+      @cmdline.delete_if {|k, v| k.to_s =~ /^help/}
+
+      # Keep track of what was passed on cmdline
+            
+      @cmdline_given = {}
+      @cmdline.keys.map do |k|
+        if k.to_s =~ /^(.+?)_given$/
+          @cmdline_given[$1.to_sym] = true
+          @cmdline.delete(k)
+        end
+      end
+      
+      @cmdline.each_pair do |k,v|
+        if @cmdline_given[k]
+          # puts "Send override #{k} = #{v}"
+          self.send(k,v) 
+        else
+          unless @config.has_key? k
+            # puts "Send default #{k} = #{v}"
+            self.send(k,v) 
+          end
+        end
+      end
+      
+      @rest = ARGV
+    end
+    
+    def [] arg
+      return @config[arg]
+    end
+    
+    def command_line_opts
+      @command = ARGV.shift # get the subcommand
+      
+      # First, deal with the help situations
+      unless SUB_COMMANDS.include? @command
+        puts "Unknown command '#{@command}'" if @command
+        print_basic_help
+      end
+      
+      if ARGV.size == 0 
+        print_basic_help
+      end
+      
+      if @command== 'help'
+        @command= ARGV.shift
+        if SUB_COMMANDS.include? @cmd
+          print_command_help @cmd
+        else
+          print_basic_help
+        end
+      end
+
+      # OK. Now let's actuall get and return the args
+      #
+      # Trollop is a DSL and doesn't see our local instance variable, so I 
+      # need to alias @commandto cmd
+      
+      cmd = @command
+      return Trollop::options do
+        OPTIONSCONFIG.each do |opt|
+          k = opt[0]
+          d = opt[1]
+          next if d[:only] and not d[:only].include? cmd.to_sym
+          desc = d.delete(:desc)
+          opt k, desc, d
+        end
+      end
+    end
+    
+    
+    def print_basic_help
+      puts %Q{
+  marc2solr: get MARC data into Solr
+  
+  USAGE
+    marc2solr index (index MARC records into Solr)
+    marc2solr delete (delete by ID from Solr)
+    marc2solr commit (send a 'commit' to a solr install)
+    
+  Use "marc2solr <cmd> --help" for more help
+
+}
+      Process.exit
+    end
+    
+    def print_command_help cmd
+      ARGV.unshift '--help'
+      Trollop::options do
+        puts "\n\n" + HELPTEXT[cmd] + "\n\n"
+        puts "You may specify multiple configuration files and they will be loaded in"
+        puts "the order given."
+        puts ""
+        puts "Command line arguments always override configuration file settings\n\n"
+        
+        OPTIONSCONFIG.each do |opt|
+          k = opt[0]
+          d = opt[1]
+          next if d[:only] and not d[:only].include? cmd.to_sym
+          desc = d.delete(:desc)
+          opt k, desc, d
+        end
+      end
+      print "\n\n"
+      Process.exit
+      
+    end
+        
+    
+    def pretty_print(pp)
+      pp.pp @config
+    end
+    
+    def method_missing(methodSymbol, arg=:notgiven, fromCmdline = false)
+      return @config[methodSymbol] if arg == :notgiven
+      methodSymbol = methodSymbol.to_s.gsub(/=$/, '').to_sym
+      
+      # Deal with negatives. We only want them if the argument is true
+      if methodSymbol.to_s =~ /^NO(.*)/
+        if arg == true
+          methodSymbol = $1.to_sym
+          arg = false
+        else
+          # puts "Ignoring false-valued #{methodSymbol}"
+          return # do nothing
+        end
+      end
+      
+      # puts "   Setting #{methodSymbol} to #{arg}"
+      if VALIDOPTIONS.has_key? methodSymbol
+        conf = VALIDOPTIONS[methodSymbol]
+        # Zero it out?
+        if conf[:takesNone] and arg.to_a.map{|a| a.downcase}.include? 'none'
+          @config[methodSymbol] = nil 
+          return nil
+        end
+        
+        
+        # Check for a valid value
+        if conf[:valid]
+          unless conf[:valid].include? arg
+            raise ArgumentError "'#{arg}' is not a valid value for #{methodSymbol}"
+          end
+        end
+        
+        # Make it a file?
+        
+        if conf[:isOutfile]
+          # If it's an IO object, just take it
+          break if arg.is_a? IO or arg.is_a? StringIO
+          
+          # Otherwise...
+          case arg.downcase
+          when "stdin"
+            arg = STDIN
+          when "stdout"
+            arg = STDOUT
+          when "stderr"
+            arg = STDERR
+          else
+            arg = File.new(arg, 'w')
+            Trollop.die "Can't open '#{arg}' for writing in argument #{methodSymbol}" unless arg
+          end
+        end
+            
+        
+        if conf[:multi]
+          @config[methodSymbol] ||= []
+          @config[methodSymbol] << arg
+          @config[methodSymbol].flatten!
+        else
+          @config[methodSymbol] = arg 
+        end
+        # puts "Set #{methodSymbol} to #{arg}"
+        return @config[methodSymbol]
+      else
+        raise NoMethodError, "'#{methodSymbol} is not a valid MARC2Solr configuration option for #{@cmd}"
+      end
+    end
+    
+    
+    # Create a SUSS from the given arguments
+    
+    def sussURL
+      machine = self[:machine] 
+      unless machine 
+        log.error  "Need solr machine name (--machine)"
+        raise ArgumentError, "Need solr machine name (--machine)"
+      end
+      
+      port = self[:port] 
+      unless port
+        log.error "Need solr port (--port)"
+        raise ArgumentError, "Need solr port (--port)"
+      end
+      
+      path = self[:solrpath]
+      unless path
+        log.error "Need solr path (--solrpath)"
+        raise ArgumentError, "Need solr path (--solrpath)"
+      end
+      
+      url = 'http://' + machine + ':' + port + '/' + path.gsub(/^\//, '')
+    end
+    
+    def suss
+      url = self.sussURL
+      log.debug "Set suss url to #{url}"
+
+      suss = StreamingUpdateSolrServer.new(url,@config[:susssize],@config[:sussthreads])
+      if self[:javabin]
+        suss.setRequestWriter Java::org.apache.solr.client.solrj.impl.BinaryRequestWriter.new
+        log.debug "Using javabin"
+      end
+      return suss
+    end
+      
+    def masterLogger
+      mlog = Logback::Simple::Logger.singleton(self.command)
+      mlog.loglevel = @config[:loglevel].downcase.to_sym
+
+      firstfile = self.rest[0] || self.command
+      logfilename = File.basename(firstfile).gsub(/\..*$/, '') # remove the last extension
+      logfilename += '-' +  Time.new.strftime('%Y%m%d-%H%M%S') + '.log'
+
+      Logback::Simple.loglevel = @config[:loglevel].downcase.to_sym
+      case @config[:logfile]
+      when "STDERR"
+        Logback::Simple.startConsoleLogger
+      when "DEFAULT"
+        Logback::Simple.startFileLogger(logfilename)
+      when 'NONE', nil
+        # do nothing
+      else
+        Logback::Simple.startFileLogger(@config[:logfile])
+      end
+      return mlog
+    end
+    
+    
+    def reader filename
+      configuredType = @config[:marctype].downcase.to_sym
+      encoding = @config[:encoding].downcase.to_sym
+      
+      if encoding == :bestguess
+        encoding = nil
+      end
+      
+      gzipped = false
+      if configuredType == :bestguess
+        if filename =~ /\.(.+)$/ # if there's an extension
+          ext = File.basename(filename).split(/\./)[-1].downcase
+          if ext == 'gz'
+            ext  = File.basename(filename).split(/\./)[-2].downcase
+            gzipped = true
+          end          
+          
+          log.info "Sniffed marc file type as #{ext}"
+          case ext
+          when /xml/, /marcxml/
+            type = :marcxml
+          when /seq/, /aleph/
+            type = :alephsequential
+          else
+            type = :permissivemarc
+          end
+        else
+          type = :permissivemarc
+        end
+      else
+        type = configuredType
+      end
+
+      source = filename
+      if source == "STDIN"
+        source = STDIN
+      end
+
+      if gzipped or @config[:gzipped]
+        source = Java::java.util.zip.GZIPInputStream.new(IOConvert.byteinstream(source))
+      end
+      
+      return MARC4J4R::Reader.new(source, type, encoding)
+    end
+    
+    
+  end
+end 
+
+    

data/spec/marc2solr_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Marc2solr" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'marc2solr'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end