marc2solr 0.1.8 → 0.2.0

data/CHANGES

@@ -1,3 +1,15 @@
+0.2.0
+  Added ability to specify custom options in config file (custom "myKey", "myvalue") or 
+  on the command line (--custom key="value")
+
+  Putting options hash in each record as it gets processed; allows you to access the options
+  in a custom function with, e.g., r.cachespot["myKey"]
+  
+0.1.9
+  Added 'ping' method
+  Changed commit/delete/index to internally call 'ping' to make sure machine is alive
+  unless dryrun is set.
+
 0.1.8
   Missed a ref to 'logback-simple'; removed.
   

data/VERSION

@@ -1 +1 @@
-0.1.8
+0.2.0

data/bin/marc2solr

@@ -9,33 +9,65 @@ require 'threach'
 
 
 
-
 opts = MARC2Solr::Conf.new
 
-# pp opts
 
 
-###########################
-# Get a master logger
-###########################
 
+# Perform the command
 
-LOG = opts.masterLogger
+# If we're going to be talking to Solr, make sure it's there!
+HC = Java::org.apache.commons.httpclient
 
-# Crank down the SUSS logging
+def pingSolr(opts)
+  if opts[:dryrun]
+    LOG.info "Dry run: not bothing to check and see if solr is alive"
+  else
+    url = opts.sussURL + '/admin/ping'
+    client = HC.HttpClient.new 
+    method = HC.methods.GetMethod.new(url)
+    begin
+      code = client.executeMethod(method)
+    rescue 
+      LOG.error "Can't ping solr URL '#{url}'" if defined? LOG
+      raise ArgumentError, "(Not up?) Can't ping solr server at #{opts.sussURL}"
+    end
+    unless code == 200
+      LOG.error "Can't ping solr URL '#{url}'" if defined? LOG
+      raise ArgumentError, "Got response code #{code} from #{opts.sussURL}"
+    end
+  end
+end  
 
-susslog = RJack::Logback['org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer']
-susslog.level = RJack::Logback::WARN
 
-# Perform the command
 
 case opts.command
+
+###################
+# Ping -- see if your solr is alive
+###################
+
+when "ping"
+  begin
+    pingSolr(opts)
+    puts "\n#{opts.sussURL} is alive\n"
+  rescue ArgumentError => e
+    puts e.message
+  end
   
 #################
 # Commit -- just send a commit to the configured solr
 #################
   
 when "commit"
+  LOG = opts.masterLogger
+
+  # Crank down the SUSS logging
+
+  susslog = RJack::Logback['org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer']
+  susslog.level = RJack::Logback::WARN
+    
+  pingSolr(opts)
   LOG.info "Commit to #{opts.sussURL}"
   if opts[:dryrun]
     LOG.debug "Using javabin" if opts[:javabin]
@@ -51,6 +83,13 @@ when "commit"
 #####################################################  
 
 when "delete"
+  LOG = opts.masterLogger
+
+  # Crank down the SUSS logging
+
+  susslog = RJack::Logback['org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer']
+  susslog.level = RJack::Logback::WARN
+    
   delfiles = opts.rest
   unless delfiles.size > 0
     LOG.error "command 'delete' needs at least one filename"
@@ -101,6 +140,14 @@ when "delete"
 #####################################################  
 
 when "index"
+
+  LOG = opts.masterLogger
+
+  # Crank down the SUSS logging
+
+  susslog = RJack::Logback['org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer']
+  susslog.level = RJack::Logback::WARN
+    
   marcfiles = opts.rest
   
   unless marcfiles.size > 0
@@ -215,6 +262,7 @@ when "index"
     reader = opts.reader(filename)
     reader.send(method, *args) do |r, i|
       Thread.current[:index] = i
+      r.cachespot[opts] = opts
       
       LOG.debug "Record {} with 001 {}", i, r['001'].value
             

data/lib/marc2solr.rb

@@ -11,10 +11,16 @@ module MARC2Solr
   class Conf
     include JLogger::Simple
     
-    SUB_COMMANDS = %w(index delete commit help)
+    SUB_COMMANDS = %w(index delete commit help ping)
     
     
     OPTIONSCONFIG = [
+      [:custom,    {:desc => "Any custom value you want. In a config file, use two String arguments (custom key value); on the command line use (--custom key=value) or (--custom key=\"three word value\")",
+                    :type=>String,
+                    :multi => true,
+                    :only => [:index],
+                    :short => '-C'
+      }],
       [:config,    {:desc => "Configuation file specifying options. Repeatable. Command-line arguments always override the config file(s)",
                     :type => :io,
                     :multi => true}],
@@ -58,7 +64,7 @@ module MARC2Solr
                   :only => [:index]
                   }],
       [:skipcommit,    {:desc => "DON'T send solr a 'commit' afterwards", 
-                    :short => '-C',
+                    :short => '-S',
                     :only => [:delete, :index],
                     }],
       [:threads,   {:desc => "Number of threads to use to process MARC records (>1 => use 'threach')", 
@@ -70,7 +76,6 @@ module MARC2Solr
                       :type => :int,
                       :default => 1}],
       [:susssize,    {:desc => "Size of the documente queue for sending to Solr", 
-                      :short => '-S',
                       :default => 128}],
       [:machine, {:desc => "Name of solr machine (e.g., solr.myplace.org)",
                       :short => '-m',
@@ -175,7 +180,7 @@ module MARC2Solr
       
       @cmdline.each_pair do |k,v|
         if @cmdline_given[k]
-          # puts "Send override #{k} = #{v}"
+          puts "Send override #{k} = #{v}"
           self.send(k,v) 
         else
           unless @config.has_key? k
@@ -274,6 +279,20 @@ module MARC2Solr
       pp.pp @config
     end
     
+    # Handle custom events specially
+    def custom (*args)
+      if args.size == 2 # called in a config file
+        @config[args[0]] = args[1]
+      else # parse it out
+        args[0].each do |str|
+          key,val = str.split(/\s*=\s*/)
+          val.gsub!(/^["']*(.*?)['"]$/, '\1')
+          @config[key] = val
+        end
+      end
+    end
+    
+    
     def method_missing(methodSymbol, arg=:notgiven, fromCmdline = false)
       return @config[methodSymbol] if arg == :notgiven
       methodSymbol = methodSymbol.to_s.gsub(/=$/, '').to_sym
@@ -362,8 +381,10 @@ module MARC2Solr
         log.error "Need solr path (--solrpath)"
         raise ArgumentError, "Need solr path (--solrpath)"
       end
+      path.gsub! /^\/*(.*?)\/*$/, '\1' # remove any leading/trailing slashes
+      path.squeeze! '/'     # make sure there are no double-slashses
       
-      url = 'http://' + machine + ':' + port.to_s + '/' + path.gsub(/^\//, '')
+      url = 'http://' + machine + ':' + port.to_s + '/' + path
     end
     
     def suss

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
     - 0
-    - 1
-    - 8
-  version: 0.1.8
+    - 2
+    - 0
+  version: 0.2.0
 platform: ruby
 authors: 
   - BillDueber
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-28 00:00:00 -04:00
+date: 2010-11-09 00:00:00 -05:00
 default_executable: 
 dependencies: 
   - !ruby/object:Gem::Dependency