traject 0.0.2 → 0.9.1

data/lib/traject/thread_pool.rb

@@ -0,0 +1,154 @@
+module Traject
+  # An abstraction wrapping a threadpool executor in some configuration choices
+  # and other apparatus. 
+  #
+  # 1) Initialize with chosen pool size -- we create fixed size pools, where 
+  #    core and max sizes are the same. 
+
+  # 2) If initialized with nil for threadcount,  no thread pool will actually
+  #    be created, and all threadpool-related methods become no-ops. We call this 
+  #    the nil/null threadpool.  A non-nil threadpool requires jruby, but you can
+  #    create a null Traject::ThreadPool.new(nil) under MRI without anything
+  #    complaining. 
+  #
+  # 3) Use the #maybe_in_threadpool method to send blocks to thread pool for
+  #    execution -- if no threadpool configured your block will just be
+  #    executed in calling thread. Be careful to not refer to any non-local
+  #    variables in the block, unless the variable has an object you can
+  #    use thread-safely! 
+  #
+  # 4) Thread pools are java.util.concurrent.ThreadPoolExecutor, manually created
+  #    with a work queue that will buffer up to (pool_size*3) tasks. If queue is full,
+  #    the ThreadPoolExecutor is set up to use the ThreadPoolExecutor.CallerRunsPolicy,
+  #    meaning the block will end up executing in caller's own thread. With the kind
+  #    of work we're doing, where each unit of work is small and there are many of them--
+  #    the CallerRunsPolicy serves as an effective 'back pressure' mechanism to keep
+  #    the work queue from getting too large and exhausting memory, when producers are
+  #    faster than consumers. 
+  #
+  #  5) Any exceptions raised by pool-executed work are captured accumulated in a thread-safe
+  #     manner, and can be re-raised in the thread of your choice by calling
+  #     #raise_collected_exception!
+  #
+  #  6) When you are done with the threadpool, you can and must call
+  #     #shutdown_and_wait, which will wait for all current queued work
+  #     to complete, then return.  You can not give any more work to the pool
+  #     after you do this. By default it'll wait pretty much forever, which should
+  #     be fine. If you never call shutdown, the pool will keep running forever
+  #     and not allow your program to exit! 
+  #
+  #  7) We will keep track of total times a block is run in thread pool, and
+  #     total elapsed (wall) time of running all blocks, so an average_execution_ms
+  #     time can be given.  #average_execution_ms may be inaccurate if called when
+  #     threads are still executing, as it's not entirely thread safe (may get
+  #     an off by one as to total iterations)
+  class ThreadPool
+    attr_reader :pool_size, :label, :queue_capacity
+
+    # First arg is pool size, 0 or nil and we'll be a null/no-op pool
+    def initialize(pool_size)
+      unless pool_size.nil? || pool_size == 0
+        require 'java' # trigger an exception now if we're not jruby
+
+        @label = label
+
+        @pool_size = pool_size.to_i # just for reflection, we don't really need it again
+        @queue_capacity = pool_size * 3
+
+
+        blockingQueue            =  java.util.concurrent.ArrayBlockingQueue.new(@queue_capacity)
+        rejectedExecutionHandler =  java.util.concurrent.ThreadPoolExecutor::CallerRunsPolicy.new
+
+        # keepalive times don't matter, we are setting core and max pool to
+        # same thing, fixed size pool. 
+        @thread_pool =  java.util.concurrent.ThreadPoolExecutor.new(
+          @pool_size, @pool_size, 0, java.util.concurrent.TimeUnit::MILLISECONDS, 
+          blockingQueue, rejectedExecutionHandler)
+
+        # A thread-safe queue to collect exceptions cross-threads. 
+        # We make it small, we really only need to store the first
+        # exception, we don't care too much about others. But we'll
+        # keep the first 20, why not. 
+        @async_exception_queue   =  java.util.concurrent.ArrayBlockingQueue.new(20)
+      end
+    end
+
+    # Pass it a block, MAYBE gets executed in the bg in a thread pool. Maybe
+    # gets executed in the calling thread. 
+    #
+    # There are actually two 'maybes':
+    # 
+    # * If Traject::ThreadPool was configured with null thread pool, then ALL
+    #   work will be executed in calling thread. 
+    #
+    # * If there is a thread pool, but it's work queue is full, then a job
+    #   will be executed in calling thread (because we configured our java
+    #   thread pool with a limited sized queue, and CallerRunsPolicy rejection strategy)
+    def maybe_in_thread_pool
+      start_t = Time.now
+
+      if @thread_pool
+        
+        @thread_pool.execute do
+          begin
+            yield
+          rescue Exception => e
+            collect_exception(e)
+          end
+        end
+      else
+        yield
+      end
+
+    end
+
+    # Just for monitoring/debugging purposes, we'll return the work queue
+    # used by the threadpool. Don't recommend you do anything with it, as
+    # the original java.util.concurrent docs make the same recommendation. 
+    def queue
+      @thread_pool && @thread_pool.queue
+    end
+
+    # thread-safe way of storing an exception, to raise
+    # later in a different thread. We don't guarantee
+    # that we can store more than one at a time, only
+    # the first one recorded may be stored. 
+    def collect_exception(e)
+      # offer will silently do nothing if the queue is full, that's fine
+      # with us. 
+      @async_exception_queue.offer(e)
+    end
+
+    # If there's a stored collected exception, raise it
+    # again now. Call this to re-raise exceptions caught in
+    # other threads in the thread of your choice. 
+    #
+    # If you call this method on a ThreadPool initialized with nil
+    # as a non-functioning threadpool -- then this method is just
+    # a no-op. 
+    def raise_collected_exception!
+      if @async_exception_queue && e = @async_exception_queue.poll
+        raise e
+      end
+    end
+
+    # shutdown threadpool, and wait for all work to complete.
+    # this one is also a no-op if you have a null ThreadPool that
+    # doesn't really have a threadpool at all. 
+    #
+    # returns elapsed time in seconds it took to shutdown
+    def shutdown_and_wait
+      start_t = Time.now
+
+      if @thread_pool
+        @thread_pool.shutdown
+        # We pretty much want to wait forever, although we need to give
+        # a timeout. Okay, one day!
+        @thread_pool.awaitTermination(1, java.util.concurrent.TimeUnit::DAYS)
+      end
+
+      return (Time.now - start_t)
+    end
+
+  end
+end

data/lib/traject/translation_map.rb

@@ -13,17 +13,18 @@ module Traject
   # or array of strings.
   #
   # What makes it more useful than a stunted hash is it's ability to load
-  # the hash definitions from configuration files, either pure ruby or
-  # yaml.
+  # the hash definitions from configuration files, either pure ruby, 
+  # yaml, or java .properties.  (Limited basic .properties, don't try any fancy escaping please,
+  # no = or : in key names, no split lines.)
   #
   # TranslationMap.new("dir/some_file")
   #
   # Will look through the entire ruby $LOAD_PATH, for a translation_maps subdir
-  # that contains either some_file.rb OR  some_file.yaml 
+  # that contains either some_file.rb OR  some_file.yaml OR some_file.properties. 
   # * Looks for "/translation_maps" subdir in load paths, so
   #   for instance you can have a gem that keeps translation maps
   #   in ./lib/translation_maps, and it Just Works. 
-  # * Note you do NOT supply the ".rb" or ".yaml" suffix yourself,
+  # * Note you do NOT supply the .rb, .yaml, or .properties suffix yourself,
   # it'll use whichever it finds (allows calling code to not care which is used).
   #
   # Ruby files just need to have their last line eval to a hash. They file
@@ -55,6 +56,8 @@ module Traject
   # When used with the #translate_array! method, one string can be replaced by multiple values
   # (array of strings) or removed (nil)
   #
+  # There's no way to specify multiple return values in a .properties, use .yaml or .rb for that. 
+  #
   # == Caching
   # Lookup and loading of configuration files will be cached, for efficiency.
   # You can reset with `TranslationMap.reset_cache!`
@@ -92,12 +95,17 @@ module Traject
         $LOAD_PATH.each do |base|
           rb_file = File.join( base,  "translation_maps",  "#{path}.rb"  )
           yaml_file = File.join( base, "translation_maps", "#{path}.yaml"  )
+          prop_file = File.join(base, "translation_maps", "#{path}.properties" )
 
           if File.exists? rb_file
             found = eval( File.open(rb_file).read , binding, rb_file )
             break
           elsif File.exists? yaml_file
             found = YAML.load_file(yaml_file)
+            break
+          elsif File.exists? prop_file
+            found = Traject::TranslationMap.read_properties(prop_file)
+            break
           end
         end
 
@@ -180,5 +188,39 @@ module Traject
       end
     end
 
+    protected
+
+    # No built-in way to read java-style .properties, we hack it. 
+    # inspired by various hacky things found google ruby java properties parse
+    # .properties spec seems to be:
+    # http://docs.oracle.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
+    #
+    # We do NOT handle split lines, don't do that!
+    def self.read_properties(file_name)
+      hash = {}
+      i = 0
+      f = File.open(file_name)
+      f.each_line do |line|
+        i += 1
+
+        line.strip! 
+
+        # skip blank lines
+        next if line.empty? 
+
+        # skip comment lines
+        next if line =~ /^\s*[!\#].*$/
+
+        if line =~ /\A([^:=]+)[\:\=]\s*(.*)\s*\Z/
+          hash[$1.strip] = $2
+        else
+          raise IOError.new("Can't parse from #{file_name} line #{i}: #{line}")
+        end
+      end
+      f.close
+
+      return hash
+    end
+
   end
 end

data/lib/traject/util.rb

@@ -0,0 +1,30 @@
+require 'traject'
+
+module Traject
+  # Just some internal utility methods
+  module Util
+
+    def self.exception_to_log_message(e)
+      indent = "    "
+
+      msg  = indent + "Exception: " + e.class.name + ": " + e.message + "\n"
+      msg += indent + e.backtrace.first + "\n"
+
+      if (e.respond_to?(:getRootCause) && e.getRootCause && e != e.getRootCause )
+        caused_by = e.getRootCause
+        msg += indent + "Caused by\n"
+        msg += indent + caused_by.class.name + ": " + caused_by.message + "\n"
+        msg += indent + caused_by.backtrace.first + "\n"
+      end
+
+      return msg
+    end
+
+    # From ruby #caller method, you get an array. Pass one line
+    # of the array here,  get just file and line number out. 
+    def self.extract_caller_location(str)
+      str.split(':in `').first
+    end
+
+  end
+end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "0.0.2"
+  VERSION = "0.9.1"
 end

data/lib/translation_maps/lcc_top_level.yaml

@@ -0,0 +1,26 @@
+# Very crude basic mapping from top-level first-letter of LCC call
+# number to an area name.
+
+A: General Works
+B: Philosophy, Psychology, Religion
+C: Historical Sciences (Archaeology, Genealogy)
+D: World History
+E: History of the Americas (General)
+F: History of the Americas (Local)
+G: Geography, Anthropology, Recreation
+H: Social Sciences
+J: Political Science
+K: Law
+L: Education
+M: Music
+N: Fine Arts
+P: Language & Literature
+Q: Science
+R: Medicine
+S: Agriculture
+T: Technology
+U: Military Science
+V: Naval Science
+Z: Bibliography, Library Science, Information Resources
+# W is NLM call number
+W: Medicine

data/lib/translation_maps/marc_genre_007.yaml

@@ -0,0 +1,9 @@
+# attempted mapping from 007 byte 1 to format/genres
+
+'a': Map/Globe
+'d': Map/Globe
+'k': Image
+'q': Musical Score
+'r': Image
+'v': Video/Film
+'m': Video/Film

data/lib/translation_maps/marc_genre_leader.yaml

@@ -0,0 +1,22 @@
+# First try to map leader bytes 6+7 combined, then if nothing
+# try to map just leader byte 6. 
+
+# first some mappings from leader bytes 6 and 7 combined:
+
+'aa': Book
+'ab': Journal/Newspaper
+'am': Book
+'as': Journal/Newspaper
+'ta': Book
+'tm': Book
+
+# Now some from just byte 6
+'c': Musical Score
+'d': Musical Score
+'e': Map/Globe
+'f': Map/Globe
+'i': Non-musical Recording
+'j': Musical Recording
+'k': Image
+'m': Software/Data
+'g': Video/Film