omelette 0.0.1a

data/lib/omelette/importer/context.rb

@@ -0,0 +1,42 @@
+# Represents the context of a specific item being imported, passed
+# to importing logic blocks
+#
+class Omelette::Importer
+  class Context
+    def initialize(hash_init = {})
+      # TODO, argument checking for required args?
+
+      self.clipboard   = {}
+      self.output_hash = {}
+
+      hash_init.each_pair do |key, value|
+        self.send("#{key}=", value)
+      end
+
+      @skip = false
+    end
+
+    attr_accessor :clipboard, :output_hash, :logger
+    attr_accessor :import_step, :source_item, :settings, :source_item_id
+    # 1-based position in stream of processed records.
+    attr_accessor :position
+
+    # Should we be skipping this record?
+    attr_accessor :skipmessage
+
+    # Set the fact that this record should be skipped, with an
+    # optional message
+    def skip!(msg = '(no message given)')
+      @skipmessage = msg
+      @skip        = true
+    end
+
+    # Should we skip this record?
+    def skip?
+      @skip
+    end
+
+  end
+
+
+end

data/lib/omelette/importer/errors.rb

@@ -0,0 +1,40 @@
+class Omelette::Importer
+  # Arity error on a passed block
+  class ArityError < ArgumentError;
+  end
+  class NamingError < ArgumentError;
+  end
+
+  # Raised by #load_config_file when config file can not
+  # be processed.
+  #
+  # The exception #message includes an error message formatted
+  # for good display to the developer, in the console.
+  #
+  # Original exception raised when processing config file
+  # can be found in #original. Original exception should ordinarily
+  # have a good stack trace, including the file path of the config
+  # file in question.
+  #
+  # Original config path in #config_file, and line number in config
+  # file that triggered the exception in #config_file_lineno (may be nil)
+  #
+  # A filtered backtrace just DOWN from config file (not including trace
+  # from omelette loading config file itself) can be found in
+  # #config_file_backtrace
+  class ConfigLoadError < StandardError
+    # We'd have #cause in ruby 2.1, filled out for us, but we want
+    # to work before then, so we use our own 'original'
+    attr_reader :original, :config_file, :config_file_lineno, :config_file_backtrace
+
+    def initialize(config_file_path, original_exception)
+      @original              = original_exception
+      @config_file           = config_file_path
+      @config_file_lineno    = Omelette::Util.backtrace_lineno_for_config(config_file_path, original_exception)
+      @config_file_backtrace = Omelette::Util.backtrace_from_config(config_file_path, original_exception)
+      message                = "Error loading configuration file #{self.config_file}:#{self.config_file_lineno} #{original_exception.class}:#{original_exception.message}"
+
+      super(message)
+    end
+  end
+end

data/lib/omelette/importer/settings.rb

@@ -0,0 +1,106 @@
+require 'hashie'
+require 'concurrent'
+
+class Omelette::Importer
+
+  # A Hash of settings for a Omelette::Importer, which also ends up passed along
+  # to other objects Omelette::Importer interacts with.
+  #
+  # Enhanced with a few features from Hashie, to make it for
+  # instance string/symbol indifferent
+  #
+  # method #provide(key, value) is added, to do like settings[key] ||= value,
+  # set only if not already set (but unlike ||=, nil or false can count as already set)
+  #
+  # Also has an interesting 'defaults' system, meant to play along
+  # with configuration file 'provide' statements. There is a built-in hash of
+  # defaults, which will be lazily filled in if accessed and not yet
+  # set. (nil can count as set, though!).  If they haven't been lazily
+  # set yet, then #provide will still fill them in. But you can also call
+  # fill_in_defaults! to fill all defaults in, if you know configuration
+  # files have all been loaded, and want to fill them in for inspection.
+  class Settings < Hash
+    include Hashie::Extensions::MergeInitializer # can init with hash
+    include Hashie::Extensions::IndifferentAccess
+
+    def initialize(*args)
+      super
+      self.default_proc = lambda do |hash, key|
+        if self.class.defaults.has_key?(key)
+          return hash[key] = self.class.defaults[key]
+        else
+          return nil
+        end
+      end
+    end
+
+    # a cautious store, which only saves key=value if
+    # there was not already a value for #key. Can be used
+    # to set settings that can be overridden on command line,
+    # or general first-set-wins settings.
+    def provide(key, value)
+      unless has_key? key
+        store(key, value)
+      end
+    end
+
+    # reverse_merge copied from ActiveSupport, pretty straightforward,
+    # modified to make sure we return a Settings
+    def reverse_merge(other_hash)
+      self.class.new(other_hash).merge(self)
+    end
+
+    def reverse_merge!(other_hash)
+      replace(reverse_merge(other_hash))
+    end
+
+    def fill_in_defaults!
+      self.reverse_merge!(self.class.defaults)
+    end
+
+
+    def self.defaults
+      {
+          # Reader defaults
+          'reader_class_name' => 'Omelette::XmlReader',
+          'marc_source.type' => 'binary',
+
+          # Writer defaults
+          'writer_class_name' => 'Omelette::OmekaJsonWriter',
+          'omeka_writer.thread_pool' => 1,
+
+          # Threading and logging
+          'processing_thread_pool' => self.default_processing_thread_pool,
+          'log.batch_size.severity' => 'info',
+
+          # how to post-process the accumulator
+          'allow_nil_values' => false,
+          'allow_duplicate_values' => true,
+
+          'allow_empty_elements' => false
+      }
+    end
+
+    def inspect
+      # Keep any key ending in password out of the inspect
+      self.inject({}) do |hash, (key, value)|
+        if /password\Z/.match(key)
+          hash[key] = '[hidden]'
+        else
+          hash[key] = value
+        end
+        hash
+      end.inspect
+    end
+
+    protected
+    def self.default_processing_thread_pool
+      if ['jruby', 'rbx'].include? ENV['RUBY_ENGINE']
+        [1, Concurrent.processor_count - 1].max
+      else
+        1
+      end
+    end
+
+  end
+end

data/lib/omelette/importer/steps.rb

@@ -0,0 +1,64 @@
+class Omelette::Importer
+  class ToElementStep
+    attr_accessor :element_name, :element_set_name, :element_id, :block, :source_location, :element_map
+    attr_reader :lambda
+
+    def initialize(element_name, element_set_name, element_map, lambda, block, source_location)
+      self.element_name = element_name
+      self.element_set_name = element_set_name
+      self.lambda = lambda
+      self.block = block
+      self.source_location = source_location
+      validate!
+
+      self.element_id = element_map[self.element_set_name][self.element_name]
+    end
+
+    def validate!
+      if self.element_name.nil? || !self.element_name.is_a?(String) || self.element_name.empty?
+        raise NamingError.new("to_element requires the element name (as a string) as the first argument at #{self.source_location}")
+      end
+      if self.element_set_name.nil? || !self.element_set_name.is_a?(String) || self.element_set_name.empty?
+        raise NamingError.new("to_element requires the element set name (as a string) as the second argument at #{self.source_location}")
+      end
+
+      [self.lambda, self.block].each do |proc|
+        if proc && (proc.arity < 2 || proc.arity > 3)
+          raise ArityError.new("error parsing element '#{element_name}': block/proc given to to_element needs 2 or 3 (or variable) arguments #{proc}, (#{self.inspect})")
+        end
+      end
+    end
+
+    def to_element_step?
+      true
+    end
+
+    def lambda=(lam)
+      @lambda       = lam
+      @lambda_arity = @lambda ? @lambda.arity : 0
+    end
+
+    def inspect
+      "(to_element #{self.element_name} at #{self.source_location})"
+    end
+
+    # to_element ""
+    def execute(context)
+      accumulator = []
+      item = context.source_item
+      if @lambda
+        if @lambda_arity == 2
+          @lambda.call item, accumulator
+        else
+          @lambda.call item, accumulator, context
+        end
+      end
+
+      if @block
+        @block.call(item, accumulator, context)
+      end
+
+      return accumulator
+    end
+  end
+end

data/lib/omelette/macros/xpath.rb

@@ -0,0 +1,14 @@
+module Omelette::Macros
+  module Xpath
+    def extract_xpath(xpath, options={})
+      options[:html] = false unless options.has_key? :html
+      lambda do |item, elements, context|
+        nodes = item.xpath xpath
+        nodes.map { |node|
+          elements << { html: options[:html], element: { id: context.import_step.element_id }, text: node.to_s.strip }
+        }
+      end
+    end
+    module_function :extract_xpath
+  end
+end

data/lib/omelette/null_writer.rb

@@ -0,0 +1,20 @@
+# A Null writer that does absolutely nothing with records given to it,
+# just drops em on the floor.
+class Omelette::NullWriter
+  def initialize(_arg_settings)
+  end
+
+
+  def serialize(_context)
+    # null
+  end
+
+  def put(_context)
+    # null
+  end
+
+  def close
+    # null
+  end
+
+end

data/lib/omelette/omeka_json_writer.rb

@@ -0,0 +1,9 @@
+# Write to Omeka using the API.
+class Omelette::OmekaJsonWriter
+  attr_reader :settings, :thread_pool_size
+  attr_reader :batched_queue
+
+  def initialize(arg_settings)
+    @settings = Omelette::Importer::Settings.new arg_settings
+  end
+end

data/lib/omelette/thread_pool.rb

@@ -0,0 +1,161 @@
+require 'concurrent'
+require 'thread' # for Queue
+
+module Omelette
+  # An abstraction wrapping a Concurrent::ThreadPool in some configuration choices
+  # and other apparatus.  Concurrent::ThreadPool is a Java ThreadPool executor on
+  # jruby for performance, and is ruby-concurrent's own ruby implementation otherwise.
+  #
+  # 1) Initialize with chosen pool size -- we create fixed size pools, where
+  # core and max sizes are the same.
+  #
+  # 2) If initialized with nil or 0 for threadcount,  no thread pool will actually
+  # be created, and work sent to the Omelette::ThreadPool will just be executed
+  # in the caller thread. We call this a nil threadpool. One situation it can be useful
+  # is if you are running under MRI, where multi-core parallelism isn't available, so
+  # an actual threadpool may not be useful. (Although in some cases a thread pool,
+  # especially one with size 1, can be useful in MRI for I/O blocking operations)
+  #
+  # 3) Use the #maybe_in_threadpool method to send blocks to thread pool for
+  # execution -- if configurred with a nil threadcount, your block will just be
+  # executed in calling thread. Be careful to not refer to any non-local
+  # variables in the block, unless the variable has an object you can
+  # use thread-safely!
+  #
+  # 4) We configure our underlying Concurrent::ThreadPool
+  # with a work queue that will buffer up to (pool_size*3) tasks. If the queue is full,
+  # the underlying Concurrent::ThreadPool is set up to use the :caller_runs policy
+  # meaning the block will end up executing in caller's own thread. With the kind
+  # of work we're doing, where each unit of work is small and there are many of them--
+  # the :caller_runs policy serves as an effective 'back pressure' mechanism to keep
+  # the work queue from getting too large and exhausting memory, when producers are
+  # faster than consumers.
+  #
+  # 5) Any exceptions raised by pool-executed work are captured accumulated in a thread-safe
+  #  manner, and can be re-raised in the thread of your choice by calling
+  #  #raise_collected_exception!
+  #
+  # 6) When you are done with the threadpool, you can and must call
+  #  #shutdown_and_wait, which will wait for all current queued work
+  #  to complete, then return.  You can not give any more work to the pool
+  #  after you do this. By default it'll wait pretty much forever, which should
+  #  be fine. If you never call shutdown, then queued or in-progress work
+  #  may be abandoned when the program ends, which would be bad.
+  #
+  # 7) We will keep track of total times a block is run in thread pool, and
+  #  total elapsed (wall) time of running all blocks, so an average_execution_ms
+  #  time can be given.  #average_execution_ms may be inaccurate if called when
+  #  threads are still executing, as it's not entirely thread safe (may get
+  #  an off by one as to total iterations)
+  class ThreadPool
+    attr_reader :pool_size, :queue_capacity
+
+    # First arg is pool size, 0 or nil and we'll be a null/no-op pool which executes
+    # work in caller thread.
+    def initialize(pool_size)
+      @thread_pool             = nil # assume we don't have one
+      @exceptions_caught_queue = [] # start off without exceptions
+      unless pool_size.nil? || pool_size == 0
+        @pool_size      = pool_size.to_i
+        @queue_capacity = pool_size * 3
+
+        @thread_pool             = Concurrent::ThreadPoolExecutor.new(
+            :min_threads     => @pool_size,
+            :max_threads     => @pool_size,
+            :max_queue       => @queue_capacity,
+            :fallback_policy => :caller_runs
+        )
+
+        # A thread-safe queue to collect exceptions cross-threads.
+        # We really only need to save the first exception, but a queue
+        # is a convenient way to store a value concurrency-safely, and
+        # might as well store all of them.
+        @exceptions_caught_queue = Queue.new
+      end
+    end
+
+    # Pass it a block, MAYBE gets executed in the bg in a thread pool. Maybe
+    # gets executed in the calling thread.
+    #
+    # There are actually two 'maybes':
+    #
+    # * If Omelette::ThreadPool was configured with null thread pool, then ALL
+    #   work will be executed in calling thread.
+    #
+    # * If there is a thread pool, but it's work queue is full, then a job
+    #   will be executed in calling thread (because we configured our java
+    #   thread pool with a limited sized queue, and CallerRunsPolicy rejection strategy)
+    #
+    # You can pass arbitrary arguments to the method, that will then be passed
+    # to your block -- similar to how ruby Thread.new works. This is convenient
+    # for creating variables unique to the block that won't be shared outside
+    # the thread:
+    #
+    #     thread_pool.maybe_in_thread_pool(x, y) do |x1, y1|
+    #       100.times do
+    #         something_with(x1)
+    #       end
+    #     end
+    #     x = "someting else"
+    #     # If we hadn't passed args with block, and had just
+    #     # used x in the block, it'd be the SAME x as this one,
+    #     # and would be pointing to a different string now!
+    #
+    #  Note, that just makes block-local variables, it doesn't
+    #  help you with whether a data structure itself is thread safe.
+    def maybe_in_thread_pool(*args)
+      if @thread_pool
+        @thread_pool.post do
+          begin
+            yield(*args)
+          rescue Exception => e
+            collect_exception(e)
+          end
+        end
+      else
+        yield(*args)
+      end
+
+    end
+
+
+    # thread-safe way of storing an exception, to raise
+    # later in a different thread. We don't guarantee
+    # that we can store more than one at a time, only
+    # the first one recorded may be stored.
+    def collect_exception(e)
+      @exceptions_caught_queue.push(e)
+    end
+
+    # If there's a stored collected exception, raise it
+    # again now. Call this to re-raise exceptions caught in
+    # other threads in the thread of your choice.
+    #
+    # If you call this method on a ThreadPool initialized with nil
+    # as a non-functioning threadpool -- then this method is just
+    # a no-op.
+    def raise_collected_exception!
+      unless @exceptions_caught_queue.empty?
+        e = @exceptions_caught_queue.pop
+        raise e
+      end
+    end
+
+    # shutdown threadpool, and wait for all work to complete.
+    # this one is also a no-op if you have a null ThreadPool that
+    # doesn't really have a threadpool at all.
+    #
+    # returns elapsed time in seconds it took to shutdown
+    def shutdown_and_wait
+      start_t = Time.now
+
+      if @thread_pool
+        @thread_pool.shutdown
+        @thread_pool.wait_for_termination
+      end
+
+      return (Time.now - start_t)
+    end
+
+  end
+end