traject 2.0.0-java

data/lib/traject/thread_pool.rb

@@ -0,0 +1,161 @@
+require 'concurrent'
+require 'thread' # for Queue
+
+module Traject
+  # An abstraction wrapping a Concurrent::ThreadPool in some configuration choices
+  # and other apparatus.  Concurrent::ThreadPool is a Java ThreadPool executor on
+  # jruby for performance, and is ruby-concurrent's own ruby implementation otherwise.
+  #
+  # 1) Initialize with chosen pool size -- we create fixed size pools, where
+  # core and max sizes are the same.
+  #
+  # 2) If initialized with nil or 0 for threadcount,  no thread pool will actually
+  # be created, and work sent to the Traject::ThreadPool will just be executed
+  # in the caller thread. We call this a nil threadpool. One situation it can be useful
+  # is if you are running under MRI, where multi-core parallelism isn't available, so
+  # an actual threadpool may not be useful. (Although in some cases a thread pool, 
+  # especially one with size 1, can be useful in MRI for I/O blocking operations)
+  #
+  # 3) Use the #maybe_in_threadpool method to send blocks to thread pool for
+  # execution -- if configurred with a nil threadcount, your block will just be
+  # executed in calling thread. Be careful to not refer to any non-local
+  # variables in the block, unless the variable has an object you can
+  # use thread-safely!
+  #
+  # 4) We configure our underlying Concurrent::ThreadPool
+  # with a work queue that will buffer up to (pool_size*3) tasks. If the queue is full,
+  # the underlying Concurrent::ThreadPool is set up to use the :caller_runs policy
+  # meaning the block will end up executing in caller's own thread. With the kind
+  # of work we're doing, where each unit of work is small and there are many of them--
+  # the :caller_runs policy serves as an effective 'back pressure' mechanism to keep
+  # the work queue from getting too large and exhausting memory, when producers are
+  # faster than consumers.
+  #
+  # 5) Any exceptions raised by pool-executed work are captured accumulated in a thread-safe
+  #  manner, and can be re-raised in the thread of your choice by calling
+  #  #raise_collected_exception!
+  #
+  # 6) When you are done with the threadpool, you can and must call
+  #  #shutdown_and_wait, which will wait for all current queued work
+  #  to complete, then return.  You can not give any more work to the pool
+  #  after you do this. By default it'll wait pretty much forever, which should
+  #  be fine. If you never call shutdown, then queued or in-progress work
+  #  may be abandoned when the program ends, which would be bad. 
+  #
+  # 7) We will keep track of total times a block is run in thread pool, and
+  #  total elapsed (wall) time of running all blocks, so an average_execution_ms
+  #  time can be given.  #average_execution_ms may be inaccurate if called when
+  #  threads are still executing, as it's not entirely thread safe (may get
+  #  an off by one as to total iterations)
+  class ThreadPool
+    attr_reader :pool_size, :queue_capacity
+
+    # First arg is pool size, 0 or nil and we'll be a null/no-op pool which executes
+    # work in caller thread. 
+    def initialize(pool_size)
+      unless pool_size.nil? || pool_size == 0
+        @pool_size = pool_size.to_i 
+        @queue_capacity = pool_size * 3
+
+        @thread_pool = Concurrent::ThreadPoolExecutor.new(
+          :min_threads     => @pool_size,
+          :max_threads     => @pool_size,
+          :max_queue       => @queue_capacity,
+          :fallback_policy => :caller_runs
+        )
+
+        # A thread-safe queue to collect exceptions cross-threads.
+        # We really only need to save the first exception, but a queue
+        # is a convenient way to store a value concurrency-safely, and
+        # might as well store all of them. 
+        @exceptions_caught_queue   =  Queue.new
+      end
+    end
+
+    # Pass it a block, MAYBE gets executed in the bg in a thread pool. Maybe
+    # gets executed in the calling thread.
+    #
+    # There are actually two 'maybes':
+    #
+    # * If Traject::ThreadPool was configured with null thread pool, then ALL
+    #   work will be executed in calling thread.
+    #
+    # * If there is a thread pool, but it's work queue is full, then a job
+    #   will be executed in calling thread (because we configured our java
+    #   thread pool with a limited sized queue, and CallerRunsPolicy rejection strategy)
+    #
+    # You can pass arbitrary arguments to the method, that will then be passed
+    # to your block -- similar to how ruby Thread.new works. This is convenient
+    # for creating variables unique to the block that won't be shared outside
+    # the thread:
+    #
+    #     thread_pool.maybe_in_thread_pool(x, y) do |x1, y1|
+    #       100.times do
+    #         something_with(x1)
+    #       end
+    #     end
+    #     x = "someting else"
+    #     # If we hadn't passed args with block, and had just
+    #     # used x in the block, it'd be the SAME x as this one,
+    #     # and would be pointing to a different string now!
+    #
+    #  Note, that just makes block-local variables, it doesn't
+    #  help you with whether a data structure itself is thread safe.
+    def maybe_in_thread_pool(*args)
+      start_t = Time.now
+
+      if @thread_pool
+        @thread_pool.post do
+          begin
+            yield(*args)
+          rescue Exception => e
+            collect_exception(e)
+          end
+        end
+      else
+        yield(*args)
+      end
+
+    end
+
+
+    # thread-safe way of storing an exception, to raise
+    # later in a different thread. We don't guarantee
+    # that we can store more than one at a time, only
+    # the first one recorded may be stored.
+    def collect_exception(e)
+      @exceptions_caught_queue.push(e)
+    end
+
+    # If there's a stored collected exception, raise it
+    # again now. Call this to re-raise exceptions caught in
+    # other threads in the thread of your choice.
+    #
+    # If you call this method on a ThreadPool initialized with nil
+    # as a non-functioning threadpool -- then this method is just
+    # a no-op.
+    def raise_collected_exception!
+      if @exceptions_caught_queue && (! @exceptions_caught_queue.empty?)
+        e = @exceptions_caught_queue.pop
+        raise e
+      end
+    end
+
+    # shutdown threadpool, and wait for all work to complete.
+    # this one is also a no-op if you have a null ThreadPool that
+    # doesn't really have a threadpool at all.
+    #
+    # returns elapsed time in seconds it took to shutdown
+    def shutdown_and_wait
+      start_t = Time.now
+
+      if @thread_pool
+        @thread_pool.shutdown
+        @thread_pool.wait_for_termination
+      end
+
+      return (Time.now - start_t)
+    end
+
+  end
+end

data/lib/traject/translation_map.rb

@@ -0,0 +1,267 @@
+require 'traject'
+
+require 'yaml'
+require 'dot-properties'
+
+
+module Traject
+  # A TranslationMap is basically just something that has a hash-like #[]
+  # method to map from input strings to output strings:
+  #
+  #    translation_map["some_input"] #=> some_output
+  #
+  # Input is assumed to always be string, output is either string
+  # or array of strings.
+  #
+  # What makes it more useful than a stunted hash is it's ability to load
+  # the hash definitions from configuration files, either pure ruby,
+  # yaml, or java .properties file (not all .properties features may
+  # be supported, we use dot-properties gem for reading)
+  #
+  # traject's `extract_marc` macro allows you to specify a :translation_map=>filename argument
+  # that will automatically find and use a translation map on the resulting data:
+  #
+  #     extract_marc("040a", :translation_map => "languages")
+  #
+  # Or you can always create one yourself and use it how you like:
+  #
+  #     map = TranslationMap.new("languages")
+  #
+  # In either case, TranslationMap will look for a file named, in that example,
+  # `languages.rb` or `languages.yaml` or `languages.properties`,
+  # somewhere in the ruby $LOAD_PATH in a `/translation_maps` subdir.
+  #
+  # * Also looks for "/translation_maps" subdir in load paths, so
+  #   for instance you can have a gem that keeps translation maps
+  #   in ./lib/translation_maps, and it Just Works.
+  #
+  # * Note you do NOT supply the .rb, .yaml, or .properties suffix yourself,
+  #  it'll use whichever it finds (allows calling code to not care which is used).
+  #
+  # Ruby files just need to have their last line eval to a hash. They file
+  # will be run through `eval`, don't do it with untrusted content (naturally)
+  #
+  # You can also pass in a Hash for consistency to TranslationMap.new, although
+  # I don't know why you'd want to.
+  #
+  # ## Special default handling
+  #
+  # The key "__default__" in the hash is treated specially. If set to a string,
+  # that string will be returned by the TranslationMap for any input not otherwise
+  # included. If set to the special string "__passthrough__", then for input not
+  # mapped, the original input string will be returned.
+  #
+  # This is most useful for YAML definition files, if you are using an actual ruby
+  # hash, you could just set the hash to do what you want using Hash#default_proc
+  # etc.
+  #
+  # Or, when calling TranslationMap.new(), you can pass in options over-riding special
+  # key too:
+  #
+  #    TranslationMap.new("something", :default => "foo")
+  #    TranslationMap.new("something", :default => :passthrough)
+  #
+  # ## Output: String or array of strings
+  #
+  # The output can be a string or an array of strings, or nil.  It should not be anything else.
+  # When used with the #translate_array! method, one string can be replaced by multiple values
+  # (array of strings) or removed (nil)
+  #
+  # There's no way to specify multiple return values in a .properties, use .yaml or .rb for that.
+  #
+  # ## Caching
+  #
+  # Lookup and loading of configuration files will be cached, for efficiency.
+  # You can reset with `TranslationMap.reset_cache!`
+  #
+  # ## YAML example:
+  #
+  #     key: value
+  #     key2: value2 multiple words fine
+  #     key2b: "Although you can use quotes if you want: Or need."
+  #     key3:
+  #       - array
+  #       - of
+  #       - values look like this
+  #
+  # ## Alternatives
+  # `Traject::TranslationMap` provides an easy way to deal with the most common translation case:
+  # simple key-value stores with optional default values.
+  #
+  # If you need more complex translation, you can simply use `#map!`
+  # or its kin to work on the `accumulator` in a block
+  #
+  #
+  #
+  #     # get a lousy language detection of any vernacular title
+  #     require 'whatlanguage'
+  #     wl = WhatLanguage.new(:all)
+  #     to_field 'vernacular_langauge', extract_marc('245', :alternate_script=>:only) do |rec, acc|
+  #       # accumulator is already filled with the values of any 880s that reference a 245 because
+  #       # of the call to #extract_marc
+  #       acc.map! {|x| wl.language(x) }
+  #       acc.uniq!
+  #     end
+  # Within the block, you may also be interested in using:
+  # * a case-insentive hash, perhaps like [this one](https://github.com/junegunn/insensitive_hash)
+  # * a [MatchMap](https://github.com/billdueber/match_map), which implements pattern-matching logic similar to solrmarc's pattern files
+  class TranslationMap
+    class Cache
+      def initialize
+        @cached = Hash.new
+      end
+
+      # Returns an actual Hash -- or nil if none found.
+      def lookup(path)
+        unless @cached.has_key?(path)
+          @cached[path] = _lookup!(path)
+        end
+        return @cached[path]
+      end
+
+      # force lookup, without using cache.
+      # used by cache. Returns the actual hash.
+      # Returns nil if none found.
+      # May raise on syntax error in file being loaded.
+      def _lookup!(path)
+        found = nil
+
+        $LOAD_PATH.each do |base|
+          rb_file = File.join( base,  "translation_maps",  "#{path}.rb"  )
+          yaml_file = File.join( base, "translation_maps", "#{path}.yaml"  )
+          prop_file = File.join(base, "translation_maps", "#{path}.properties" )
+
+          if File.exists? rb_file
+            found = eval( File.open(rb_file).read , binding, rb_file )
+            break
+          elsif File.exists? yaml_file
+            found = YAML.load_file(yaml_file)
+            break
+          elsif File.exists? prop_file
+            found = Traject::TranslationMap.read_properties(prop_file)
+            break
+          end
+        end
+
+        # Cached hash can't be mutated without weird consequences, let's
+        # freeze it!
+        found.freeze if found
+
+        return found
+      end
+
+      def reset_cache!
+        @cached.clear
+      end
+
+    end
+
+    attr_reader :hash
+    attr_reader :default
+
+    class << self
+      attr_accessor :cache
+      def reset_cache!
+        cache.reset_cache!
+      end
+    end
+    self.cache = Cache.new
+
+
+    def initialize(defn, options = {})
+      if defn.kind_of? Hash
+        @hash = defn
+      elsif defn.kind_of? self.class
+        @hash = defn.to_hash
+        @default = defn.default
+      else
+        @hash = self.class.cache.lookup(defn)
+        raise NotFound.new(defn) if @hash.nil?
+      end
+
+      if options[:default]
+        @default = options[:default]
+      elsif @hash.has_key? "__default__"
+        @default = @hash["__default__"]
+      end
+    end
+
+    def [](key)
+      if self.default && (! @hash.has_key?(key))
+        if self.default == "__passthrough__"
+          return key
+        else
+          return self.default
+        end
+      end
+
+      @hash[key]
+    end
+    alias_method :map, :[]
+
+    # Returns a dup of internal hash, dup so you can modify it
+    # if you like.
+    def to_hash
+      dup = @hash.dup
+      dup.delete("__default__")
+      dup
+    end
+
+    # Run every element of an array through this translation map,
+    # return the resulting array. If translation map returns nil,
+    # original element will be missing from output.
+    #
+    # If an input maps to an array, each element of the array will be flattened
+    # into the output.
+    #
+    # If an input maps to nil, it will cause the input element to be removed
+    # entirely.
+    def translate_array(array)
+      array.each_with_object([]) do |input_element, output_array|
+        output_element = self.map(input_element)
+        if output_element.kind_of? Array
+          output_array.concat output_element
+        elsif ! output_element.nil?
+          output_array << output_element
+        end
+      end
+    end
+
+    def translate_array!(array)
+      array.replace( self.translate_array(array))
+    end
+
+    # Return a new TranslationMap that results from merging argument on top of self. 
+    # Can be useful for taking an existing translation map, but merging a few
+    # overrides on top. 
+    #
+    #     merged_map = TranslationMap.new(something).merge TranslationMap.new(else)
+    #     #...
+    #     merged_map.translate_array(something) # etc
+    #
+    # If a default is set in the second map, it will merge over the first too. 
+    #
+    # You can also pass in a plain hash as an arg, instead of an existing TranslationMap:
+    #
+    #     TranslationMap.new(something).merge("overridden_key" => "value", "a" => "")
+    def merge(other_map)
+      default = other_map.default || self.default 
+      TranslationMap.new(self.to_hash.merge(other_map.to_hash), :default => default)
+    end
+
+    class NotFound < Exception
+      def initialize(path)
+        super("No translation map definition file found at 'translation_maps/#{path}.[rb|yaml|properties]' in load path: #{$LOAD_PATH}")
+      end
+    end
+
+    protected
+
+    # We use dot-properties gem for reading .properties files,
+    # return a hash. 
+    def self.read_properties(file_name)   
+      return DotProperties.load(file_name).to_h      
+    end
+
+  end
+end

data/lib/traject/util.rb

@@ -0,0 +1,52 @@
+require 'traject'
+
+module Traject
+  # Just some internal utility methods
+  module Util
+
+    def self.exception_to_log_message(e)
+      indent = "    "
+
+      msg  = indent + "Exception: " + e.class.name + ": " + e.message + "\n"
+      msg += indent + e.backtrace.first + "\n"
+
+      if (e.respond_to?(:getRootCause) && e.getRootCause && e != e.getRootCause )
+        caused_by = e.getRootCause
+        msg += indent + "Caused by\n"
+        msg += indent + caused_by.class.name + ": " + caused_by.message + "\n"
+        msg += indent + caused_by.backtrace.first + "\n"
+      end
+
+      return msg
+    end
+
+    # From ruby #caller method, you get an array. Pass one line
+    # of the array here,  get just file and line number out.
+    def self.extract_caller_location(str)
+      str.split(':in `').first
+    end
+
+
+
+    # Ruby stdlib queue lacks a 'drain' function, we write one.
+    #
+    # Removes everything currently in the ruby stdlib queue, and returns
+    # it an array.  Should be concurrent-safe, but queue may still have
+    # some things in it after drain, if there are concurrent writers.
+    def self.drain_queue(queue)
+      result = []
+
+      queue_size = queue.size
+      begin
+        queue_size.times do
+          result << queue.deq(:raise_if_empty)
+        end
+      rescue ThreadError
+        # Need do nothing, queue was concurrently popped, no biggie
+      end
+
+      return result
+    end
+
+  end
+end