marc2solr 0.1.4 → 0.1.6

data/CHANGES

@@ -1,3 +1,9 @@
+0.1.6
+  Changed to parameterized logging with jlogger 0.0.4 for faster non-logging
+
+0.1.5
+  Changed to use jlogger
+    
 0.1.4
   Add dependency on Trollop to Rakefile
   

data/Rakefile

@@ -11,19 +11,18 @@ begin
     gem.homepage = "http://github.com/billdueber/marc2solr"
     gem.authors = ["BillDueber"]
     
-    gem.add_dependency 'marc4j4r', '>= 1.2.0'
+    gem.add_dependency 'marc4j4r', '>= 1.3.0'
     gem.add_dependency 'jruby_streaming_update_solr_server', '>=0.5.2'
-    gem.add_dependency 'marcspec', '>= 1.6.3'
+    gem.add_dependency 'marcspec', '>= 1.6.5'
     gem.add_dependency 'threach', '>= 0.2.0'
-    gem.add_dependency 'logback-simple'
+    gem.add_dependency 'jlogger', '>=0.0.4'
     gem.add_dependency 'library_stdnums', '>=0.2.0'
     gem.add_dependency 'trollop'
+    
     gem.add_development_dependency "rspec", ">= 1.2.9"
     gem.add_development_dependency "yard", ">= 0"
 
     gem.bindir = 'bin'
-    gem.executables << 'solrmarc_to_marc2solr'
-    gem.executables << 'marc2solr'
     
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.6

data/bin/marc2solr

@@ -21,7 +21,7 @@ opts = MARC2Solr::Conf.new
 ###########################
 
 
-$LOG = opts.masterLogger
+LOG = opts.masterLogger
 
 # Perform the command
 
@@ -32,13 +32,13 @@ case opts.command
 #################
   
 when "commit"
-  $LOG.info "Commit to #{opts.sussURL}"
+  LOG.info "Commit to #{opts.sussURL}"
   if opts[:dryrun]
-    $LOG.debug "Using javabin" if opts[:javabin]
-    $LOG.info "DRY RUN. Stopping now."
+    LOG.debug "Using javabin" if opts[:javabin]
+    LOG.info "DRY RUN. Stopping now."
   else
     opts.suss.commit
-    $LOG.info "Commit done"
+    LOG.info "Commit done"
   end
   
 
@@ -49,7 +49,7 @@ when "commit"
 when "delete"
   delfiles = opts.rest
   unless delfiles.size > 0
-    $LOG.error "command 'delete' needs at least one filename"
+    LOG.error "command 'delete' needs at least one filename"
     puts "\n\nERROR: command 'delete' needs at least one filename"
     opts.print_command_help('delete')
   end
@@ -60,7 +60,7 @@ when "delete"
   # Make sure they can all be opened
   delfiles.each do |filename|
     unless File.readable? filename
-      $LOG.error "Can't open configuration file `#{filename}`"
+      LOG.error "Can't open configuration file `#{filename}`"
       raise ArgumentError, "Can't open configuration file `#{filename}`" 
     end
   end
@@ -71,24 +71,24 @@ when "delete"
   delfiles.each do |filename|
     count = 0
     f = File.open(filename)
-    $LOG.info "Deleting IDs listed in #{filename}"
-    $LOG.info "DRY RUN ONLY" if opts[:dryrun]
+    LOG.info "Deleting IDs listed in #{filename}"
+    LOG.info "DRY RUN ONLY" if opts[:dryrun]
     f.each_line do |id|
       id.chomp!
       suss.deleteById(id) unless opts[:dryrun]
-      $LOG.debug "Deleted id\t#{id}"
+      LOG.debug "Deleted id\t{}", id
       count += 1
       total += 1
     end
     f.close
-    $LOG.info "Tried to delete #{count} ids from file '#{filename}'"
+    LOG.info "Tried to delete #{count} ids from file '#{filename}'"
   end
-  $LOG.info "Tried to delete #{total} ids from all #{delfiles.size} files" if delfiles.size > 1
+  LOG.info "Tried to delete #{total} ids from all #{delfiles.size} files" if delfiles.size > 1
 
   unless opts[:dryrun] or opts[:skipcommit]
-    $LOG.info "Sending final commit"
+    LOG.info "Sending final commit"
     suss.commit
-    $LOG.info "Final commit finished"
+    LOG.info "Final commit finished"
   end
 
 
@@ -100,7 +100,7 @@ when "index"
   marcfiles = opts.rest
   
   unless marcfiles.size > 0
-    $LOG.error "command 'delete' needs at least one marc file to index"
+    LOG.error "command 'delete' needs at least one marc file to index"
     puts "\n\nERROR: command 'index' needs at least one filename"
     opts.print_command_help('index')
   end
@@ -108,48 +108,48 @@ when "index"
   
   # Make sure everything can be opened
   unless File.readable? opts[:indexfile]
-    $LOG.error "Index configuration file '#{opts[:indexfile]}' (set with --indexfile) cannot be found/read"
+    LOG.error "Index configuration file '#{opts[:indexfile]}' (set with --indexfile) cannot be found/read"
     raise ArgumentError, "Index configuration file '#{opts[:indexfile]}' (set with --indexfile) cannot be found/read"
   else
-    $LOG.debug "Found index file #{opts[:indexfile]}"
+    LOG.debug "Found index file #{opts[:indexfile]}"
   end
   
   unless File.readable? opts[:tmapdir]
-    $LOG.error "Translation Map directory '#{opts[:tmapdir]}' (set with --tmapdir) cannot be found/read"
+    LOG.error "Translation Map directory '#{opts[:tmapdir]}' (set with --tmapdir) cannot be found/read"
     raise ArgumentError, "Translation Map directory '#{opts[:tmapdir]}' (set with --tmapdir) cannot be found/read"
   else
-    $LOG.debug "Found translation maps directory #{opts[:tmapdir]}"
+    LOG.debug "Found translation maps directory #{opts[:tmapdir]}"
   end
   
   marcfiles.each do |filename|
     if filename == "STDIN"
-      $LOG.info "Using standard input as a marc file"
+      LOG.info "Using standard input as a marc file"
       next
     end
     unless File.readable? filename
-      $LOG.error "Can't open MARC file `#{filename}`"
+      LOG.error "Can't open MARC file `#{filename}`"
       raise ArgumentError, "Can't open MARC file `#{filename}`" 
     else
-      $LOG.debug "Adding marc file #{filename} to queue"
+      LOG.debug "Adding marc file #{filename} to queue"
     end
   end
   
 
   # Load all the files in the customdir(s)
-  $LOG.debug "Loading custom routines"
+  LOG.debug "Loading custom routines"
   if opts[:customdir].size > 0
     opts[:customdir].each do |dir|
       unless File.exist? dir
-        $LOG.warn "Skipping load directory '#{dir}': Not found"
+        LOG.warn "Skipping load directory '#{dir}': Not found"
         opts[:customdir].delete dir
       end
-      $LOG.info "Loading files in #{dir}"
+      LOG.info "Loading files in #{dir}"
       Dir.glob(["#{dir}/*.jar"]).each do |x|
-        $LOG.debug "Loading #{x}"
+        LOG.debug "Loading #{x}"
         require x
       end
       Dir.glob(["#{dir}/*.rb"]).each do |x|
-        $LOG.debug "Loading #{x}"
+        LOG.debug "Loading #{x}"
         require x
       end
     end
@@ -166,26 +166,28 @@ when "index"
   
   # Set up if we're using threach or not
   if opts[:threads] > 1
+    LOG.info "Using #{opts[:threads]} threads; activiating threach"
     method = :threach
     args = [opts[:threads], :each_with_index]
   else
+    LOG.info "Using 1 thread"
     method = :each_with_index
     args = []
   end
   
   
   if opts[:dryrun]
-    $LOG.info "Begin DRY RUN; nothing will be sent to Solr"
+    LOG.info "Begin DRY RUN; nothing will be sent to Solr"
   end
   
   # Clear solr if so declared
   if opts[:clearsolr] 
     if opts[:dryrun]
-      $LOG.info "Would have cleared out solr (but for dryrun)"
+      LOG.info "Would have cleared out solr (but for dryrun)"
     else
       suss.deleteByQuery('*:*')
       suss.commit
-      $LOG.info "Cleared out solr"
+      LOG.info "Cleared out solr"
     end
   end
   
@@ -195,11 +197,13 @@ when "index"
   
   start = Time.new.to_f
   marcfiles.each do |filename|
-    $LOG.info "Indexing file #{filename}"
+    LOG.info "Indexing file {}", filename
     
     reader = opts.reader(filename)
     reader.send(method, *args) do |r, i|
       Thread.current[:index] = i
+      
+      LOG.debug "Record {} with 001 {}", i, r['001'].value
             
       doc = ss.doc_from_marc(r, opts[:benchmark])
       
@@ -215,7 +219,7 @@ when "index"
       
       if Thread.current[:index] % opts[:logbatchsize] == 0
         pace  = Thread.current[:index] / (Time.new.to_f - start) 
-        $LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [Thread.current[:index], pace]
+        LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [Thread.current[:index], pace]
       end
     end # reader
   end # marcfiles
@@ -223,21 +227,21 @@ when "index"
   # Commit
   
   unless opts[:dryrun] or opts[:skipcommit]
-    $LOG.info "Sending final commit"
+    LOG.info "Sending final commit"
     suss.commit
-    $LOG.info "Final commit finished"
+    LOG.info "Final commit finished"
   end
   
   # Be done
-  $LOG.info "Done indexing"
+  LOG.info "Done indexing"
   pace  = i / (Time.new.to_f - start) 
-  $LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [i, pace]
+  LOG.info "%d indexed (overall pace: %.0f rec/sec)" % [i, pace]
   
   
   # Log the benchmarking information if requested
   if opts[:benchmark]
     ss.benchmarks.keys.sort{|a,b| ss.benchmarks[b].real <=> ss.benchmarks[a].real}.each do |k|
-      $LOG.info("%-20s %s" % [k + ':', ss.benchmarks[k].real.to_s])
+      LOG.info("%-20s %s" % [k + ':', ss.benchmarks[k].real.to_s])
     end
   end
   

data/lib/marc2solr/marc2solr_custom.rb

@@ -1,10 +1,10 @@
 require 'rubygems'
-require 'logback-simple'
+require 'jlogger'
 require 'library_stdnums'
 
 module MARC2Solr
   module Custom
-    LOG = Logback::Simple::Logger.singleton
+    LOG = JLogger::RootLogger.new
     
     # Custom routines are defined as module methods that take two arguments: a MARC4J4R record,
     # and an (optional) array of other arguments passed in. 

data/lib/marc2solr.rb

@@ -1,6 +1,6 @@
 require 'rubygems'
 
-require 'logback-simple'
+require 'jlogger'
 require 'trollop'
 require 'ftools'
 require 'jruby_streaming_update_solr_server'
@@ -9,7 +9,7 @@ require 'marc4j4r'
 module MARC2Solr
   
   class Conf
-    include Logback::Simple
+    include JLogger::Simple
     
     SUB_COMMANDS = %w(index delete commit help)
     
@@ -149,7 +149,7 @@ module MARC2Solr
       # Load the config files
       if @cmdline[:config]
         @cmdline[:config].each do |f|
-          log.info "Reading config-file '#{f.path}'"
+          log.info "Reading config-file '{}'", f.path
           self.instance_eval(f.read)
         end
       end
@@ -367,7 +367,12 @@ module MARC2Solr
     
     def suss
       url = self.sussURL
-      log.debug "Set suss url to #{url}"
+      log.info "Set suss url to {}", url
+      if @config[:sussthreads] > 1
+        log.info "Using {} threads for the suss", @config[:sussthreads]
+      else
+        log.info "Using a single thread for the suss"
+      end
 
       suss = StreamingUpdateSolrServer.new(url,@config[:susssize],@config[:sussthreads])
       if self[:javabin]
@@ -378,23 +383,24 @@ module MARC2Solr
     end
       
     def masterLogger
-      mlog = Logback::Simple::Logger.singleton(self.command)
+      mlog = JLogger::RootLogger.new
       mlog.loglevel = @config[:loglevel].downcase.to_sym
 
       firstfile = self.rest[0] || self.command
       logfilename = File.basename(firstfile).gsub(/\..*$/, '') # remove the last extension
       logfilename += '-' +  Time.new.strftime('%Y%m%d-%H%M%S') + '.log'
 
-      Logback::Simple.loglevel = @config[:loglevel].downcase.to_sym
       case @config[:logfile]
       when "STDERR"
-        Logback::Simple.startConsoleLogger
+        mlog.startConsole
       when "DEFAULT"
-        Logback::Simple.startFileLogger(logfilename)
+        mlog.startFile(logfilename)
       when 'NONE', nil
+        mlog.stopConsole
+        mlog.loglevel = :off
         # do nothing
       else
-        Logback::Simple.startFileLogger(@config[:logfile])
+        mlog.startFile(@config[:logfile])
       end
       return mlog
     end
@@ -417,7 +423,7 @@ module MARC2Solr
             gzipped = true
           end          
           
-          log.info "Sniffed marc file type as #{ext}"
+          log.info "Sniffed marc file type as {}", ext
           case ext
           when /xml/, /marcxml/
             type = :marcxml

metadata

@@ -1,199 +1,222 @@
 --- !ruby/object:Gem::Specification 
 name: marc2solr
 version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: false
   segments: 
-    - 0
-    - 1
-    - 4
-  version: 0.1.4
+  - 0
+  - 1
+  - 6
+  version: 0.1.6
 platform: ruby
 authors: 
-  - BillDueber
+- BillDueber
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2010-10-13 00:00:00 -04:00
+date: 2010-10-15 00:00:00 -04:00
 default_executable: 
 dependencies: 
-  - !ruby/object:Gem::Dependency 
-    name: marc4j4r
-    prerelease: false
-    requirement: &id001 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 1
-              - 2
-              - 0
-            version: 1.2.0
-    type: :runtime
-    version_requirements: *id001
-  - !ruby/object:Gem::Dependency 
-    name: jruby_streaming_update_solr_server
-    prerelease: false
-    requirement: &id002 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-              - 5
-              - 2
-            version: 0.5.2
-    type: :runtime
-    version_requirements: *id002
-  - !ruby/object:Gem::Dependency 
-    name: marcspec
-    prerelease: false
-    requirement: &id003 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 1
-              - 6
-              - 3
-            version: 1.6.3
-    type: :runtime
-    version_requirements: *id003
-  - !ruby/object:Gem::Dependency 
-    name: threach
-    prerelease: false
-    requirement: &id004 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-              - 2
-              - 0
-            version: 0.2.0
-    type: :runtime
-    version_requirements: *id004
-  - !ruby/object:Gem::Dependency 
-    name: logback-simple
-    prerelease: false
-    requirement: &id005 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-            version: "0"
-    type: :runtime
-    version_requirements: *id005
-  - !ruby/object:Gem::Dependency 
-    name: library_stdnums
-    prerelease: false
-    requirement: &id006 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-              - 2
-              - 0
-            version: 0.2.0
-    type: :runtime
-    version_requirements: *id006
-  - !ruby/object:Gem::Dependency 
-    name: trollop
-    prerelease: false
-    requirement: &id007 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-            version: "0"
-    type: :runtime
-    version_requirements: *id007
-  - !ruby/object:Gem::Dependency 
-    name: rspec
-    prerelease: false
-    requirement: &id008 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 1
-              - 2
-              - 9
-            version: 1.2.9
-    type: :development
-    version_requirements: *id008
-  - !ruby/object:Gem::Dependency 
-    name: yard
-    prerelease: false
-    requirement: &id009 !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            segments: 
-              - 0
-            version: "0"
-    type: :development
-    version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: marc4j4r
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: jruby_streaming_update_solr_server
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 0
+        - 5
+        - 2
+        version: 0.5.2
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: marcspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 6
+        - 5
+        version: 1.6.5
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: threach
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: jlogger
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 0
+        - 4
+        version: 0.0.4
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: library_stdnums
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: yard
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id009
 description: Given a file of MARC records, send them to Sorl for indexing based on a set of MARCSpecs
 email: bill@dueber.com
 executables: 
-  - marc2solr
-  - solrmarc_to_marc2solr
-  - solrmarc_to_marc2solr
-  - marc2solr
+- marc2solr
+- solrmarc_to_marc2solr
 extensions: []
 
 extra_rdoc_files: 
-  - LICENSE
-  - README.md
+- LICENSE
+- README.md
 files: 
-  - .document
-  - .gitignore
-  - CHANGES
-  - LICENSE
-  - README.md
-  - Rakefile
-  - VERSION
-  - bin/marc2solr
-  - bin/solrmarc_to_marc2solr
-  - lib/marc2solr.rb
-  - lib/marc2solr/marc2solr_custom.rb
-  - spec/marc2solr_spec.rb
-  - spec/spec.opts
-  - spec/spec_helper.rb
+- .document
+- .gitignore
+- CHANGES
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- bin/marc2solr
+- bin/solrmarc_to_marc2solr
+- lib/marc2solr.rb
+- lib/marc2solr/marc2solr_custom.rb
+- spec/marc2solr_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/billdueber/marc2solr
 licenses: []
 
 post_install_message: 
 rdoc_options: 
-  - --charset=UTF-8
+- --charset=UTF-8
 require_paths: 
-  - lib
+- lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-          - 0
-        version: "0"
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-          - 0
-        version: "0"
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: "MARC2Solr: Get MARC into Solr via JRuby"
 test_files: 
-  - spec/marc2solr_spec.rb
-  - spec/spec_helper.rb
+- spec/marc2solr_spec.rb
+- spec/spec_helper.rb