omelette 0.0.1a

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d5c99f1e799db62ab94213b0919b401f00fa7f0f
+  data.tar.gz: 8f89f4fb46dee7a19305f20432acf7ead96aa9bf
+SHA512:
+  metadata.gz: 0d5ab5c28afe2ec4b2cd7d137813465c317ba2a3519baa287719a6fa545aaad1912fa322092de188349555c7fbd53c0264c46155842130389bd7ebfa063b3d79
+  data.tar.gz: b58aa6854cff01fbf7bd79250e128c8f69aa2c03975ee3376c735395d0626fb921589326c0d98d879861645e843c747ab9ce526e507749c32053db1635619efd

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+/.ruby-version
+/.ruby-gemset
+/.idea
+/omelette-*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+  - 2.4.0
+before_install: gem install bundler -v 1.16.0

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in omelette.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    omelette (0.0.1a)
+      concurrent-ruby (~> 1.0)
+      hashie (~> 3.5)
+      nokogiri (~> 1.8)
+      rest-client (~> 2.0)
+      thor (~> 0.20)
+      yell (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    concurrent-ruby (1.0.5)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    domain_name (0.5.20170404)
+      unf (>= 0.0.5, < 1.0.0)
+    hashdiff (0.3.7)
+    hashie (3.5.6)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.3.0)
+    netrc (0.11.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    public_suffix (3.0.1)
+    rake (10.5.0)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    safe_yaml (1.0.4)
+    thor (0.20.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.4)
+    webmock (3.1.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yell (2.0.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  omelette!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  webmock (~> 3.0)
+
+BUNDLED WITH
+   1.16.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Dazhi Jiao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Omelette
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/omelette`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'omelette'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install omelette
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/omelette. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Omelette project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/omelette/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "omelette"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/omelette

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'omelette'
+
+Omelette::CommandLine.start(ARGV)

data/lib/omelette.rb

@@ -0,0 +1,10 @@
+require 'omelette/version'
+require 'omelette/importer'
+require 'omelette/util'
+require 'omelette/macros/xpath'
+require 'omelette/xml_reader'
+require 'omelette/command_line'
+
+module Omelette
+  # Your code goes here...
+end

data/lib/omelette/command_line.rb

@@ -0,0 +1,10 @@
+require 'thor'
+
+module Omelette
+  class CommandLine < Thor
+    desc 'import', 'import a file'
+    def import(ids = nil)
+      puts 'omelette import: To be implemented'
+    end
+  end
+end

data/lib/omelette/importer.rb

@@ -0,0 +1,284 @@
+require 'yell'
+require 'omelette/importer/context'
+require 'omelette/importer/settings'
+require 'omelette/importer/errors'
+require 'omelette/importer/steps'
+require 'omelette/thread_pool'
+require 'omelette/macros/xpath'
+
+class Omelette::Importer
+  include Omelette::Macros::Xpath
+
+  attr_writer :reader_class, :writer_class, :writer
+  attr_reader :logger
+
+  def initialize(arg_settings = {})
+    @settings = Settings.new(arg_settings)
+    @import_steps = []
+    @after_processing_steps = []
+
+    @logger = create_logger
+  end
+
+  def elements_map
+    @elements_map ||= Omelette::Util.build_elements_map @settings['omeka_api_root']
+  end
+
+  # Pass a string file path, a Pathname, or a File object, for
+  # a config file to load into indexer.
+  #
+  # Can raise:
+  # * Errno::ENOENT or Errno::EACCES if file path is not accessible
+  # * Omelette::Importer::ConfigLoadError if exception is raised evaluating
+  #   the config. A ConfigLoadError has information in it about original
+  #   exception, and exactly what config file and line number triggered it.
+  def load_config_file(file_path)
+    File.open file_path do |file|
+      begin
+        self.instance_eval file.read, file_path.to_s
+      rescue ScriptError, StandardError => ex
+        raise ConfigLoadError.new(file_path.to_s, ex)
+      end
+    end
+  end
+
+  def settings(new_settings = nil, &block)
+    @settings.merge! new_settings if new_settings
+    @settings.instance_eval &block if block_given?
+    return @settings
+  end
+
+  def to_element(element_name, element_set_name, aLambda = nil, &block)
+    @import_steps << ToElementStep.new(element_name, element_set_name, elements_map, aLambda, block, Omelette::Util.extract_caller_location(caller.first))
+  end
+
+  # Processes a single item according to extracting rules set up in
+  # this importer. Returns the output hash (a hash whose keys are
+  # string fields, and values are arrays of one or more values in that field)
+  #
+  # This is a convenience shortcut for #map_to_context! -- use that one
+  # if you want to provide addtional context
+  # like position, and/or get back the full context.
+  def map_item(item)
+    context = Context.new(source_item: item, settings: settings)
+    map_to_context! context
+    return context.output_hash
+  end
+
+  def map_to_context!(context)
+    @import_steps.each do |import_step|
+      break if context.skip?
+
+      # set the to_element step for error reporting
+      context.import_step = import_step
+      elements = log_mapping_errors context, import_step do
+        import_step.execute context
+      end
+      add_elements_to_context!(elements, context) if import_step.to_element_step?
+
+      # Unset the import step after it's finished
+      context.import_step = nil
+    end
+    return context
+  end
+
+  # Add the accumulator to the context with the correct field name
+  # Do post-processing on the accumulator (remove nil values, allow empty
+  # fields, etc)
+  #
+  # Only get here if we've got a to_field step; otherwise the
+  # call to get a field_name will throw an error
+
+  ALLOW_NIL_VALUES       = 'allow_nil_values'.freeze
+  ALLOW_EMPTY_FIELDS     = 'allow_empty_fields'.freeze
+  ALLOW_DUPLICATE_VALUES = 'allow_duplicate_values'.freeze
+
+  def add_elements_to_context!(elements, context)
+    elements.compact! unless settings[ALLOW_NIL_VALUES]
+    return if elements.empty? and not (settings[ALLOW_EMPTY_FIELDS])
+
+    element_name = context.import_step.element_name
+    context.output_hash[element_name] ||= []
+
+    existing_element = context.output_hash[element_name].concat element
+    existing_element.uniq! unless settings[ALLOW_DUPLICATE_VALUES]
+
+  rescue NameError => ex
+    msg = 'Tried to call add_element_to_context with a non-to_element step'
+    msg += context.import_step.inspect
+    logger.error msg
+    raise ArgumentError.new msg
+  end
+
+  def process(files)
+    settings.fill_in_defaults!
+
+    count = 0
+    start_time = batch_start_time = Time.now
+    logger.debug "beginning Omelette::Import*process with settings: #{settings.inspect}"
+    reader = self.reader! files
+    processing_threads = settings['processing_thread_pool'].to_i
+    thread_pool = Omelette::ThreadPool.new processing_threads
+
+    logger.info "   Indexer with #{processing_threads} processing threads, reader: #{reader.class.name} and writer: #{writer.class.name}"
+    log_batch_size = settings['log.batch_size'] && settings['log.batch_size'].to_i
+
+    reader.each do |item, item_id; position|
+      count += 1
+      position = count
+
+      thread_pool.raise_collected_exception!
+
+      if settings['debug_ascii_progress'].to_s == 'true'
+        $stderr.write '.' if count % settings['solr_writer.batch_size'].to_i == 0
+      end
+
+      context = Context.new source_item: item, source_item_id: item_id, settings: settings, position: position, logger: logger
+
+      if log_batch_size && (count % log_batch_size == 0)
+        batch_rps   = log_batch_size / (Time.now - batch_start_time)
+        overall_rps = count / (Time.now - start_time)
+        logger.send(settings['log.batch_size.severity'].downcase.to_sym, "Omelette::Importer#process, read #{count} records at id:#{context.source_record_id}; #{'%.0f' % batch_rps}/s this batch, #{'%.0f' % overall_rps}/s overall")
+        batch_start_time = Time.now
+      end
+
+      thread_pool.maybe_in_thread_pool(context) do |context|
+        map_to_context! context
+        if context.skip?
+          log_skip context
+        else
+          writer.put context
+        end
+      end
+    end
+
+    $stderr.write "\n" if settings['debug_ascii_progress'].to_s == 'true'
+
+    logger.debug 'Shutting down #processing mapper threadpool...'
+    thread_pool.shutdown_and_wait
+    logger.debug '#processing mapper threadpool shutdown complete.'
+
+    thread_pool.raise_collected_exception!
+
+
+    writer.close if writer.respond_to?(:close)
+
+    @after_processing_steps.each do |step|
+      begin
+        step.execute
+      rescue Exception => e
+        logger.fatal("Unexpected exception #{e} when executing #{step}")
+        raise e
+      end
+    end
+
+    elapsed = Time.now - start_time
+    avg_rps = (count / elapsed)
+    logger.info "finished Indexer#process: #{count} records in #{'%.3f' % elapsed} seconds; #{'%.1f' % avg_rps} records/second overall."
+
+    if writer.respond_to?(:skipped_record_count) && writer.skipped_record_count > 0
+      logger.error "Indexer#process returning 'false' due to #{writer.skipped_record_count} skipped records."
+      return false
+    end
+
+    return true
+  end
+
+  def reader_class
+    unless defined? @reader_class
+      @reader_class = Object.const_get(settings['reader_class_name']) rescue nil
+    end
+    return @reader_class
+  end
+
+  def writer_class
+    writer.class
+  end
+
+  # Instantiate a Omelette Reader, using class set
+  # in #reader_class, initialized with io_stream passed in
+  def reader!(ids)
+    return reader_class.new(ids, settings.merge('logger' => logger))
+  end
+
+  # Instantiate a Writer, suing class set in #writer_class
+  def writer!
+    writer_class = @writer_class || Object.const_get(settings['writer_class_name']) rescue nil
+    writer_class.new(settings.merge('logger' => logger))
+  end
+
+  def writer
+    @writer ||= settings['writer'] || writer!
+  end
+
+  # Log that the current record is being skipped, using
+  # data in context.position and context.skipmessage
+  def log_skip(context)
+    logger.debug "Skipped record #{context.position}: #{context.skipmessage}"
+  end
+  private :log_skip
+
+  def log_mapping_errors(context, import_step)
+    begin
+      yield
+    rescue Exception => ex
+      msg = "Unexpected error on record id `#{context.source_item_id}` at file position #{context.position}\n"
+      msg += "    while executing #{import_step.inspect}\n"
+      msg += Omelette::Util.exception_to_log_message(e)
+
+      logger.error msg
+      begin
+        logger.debug "Item: #{context.source_item.to_s}"
+      rescue Exception => item_exception
+        logger.debug "(Could not log item, #{item_exception})"
+      end
+      raise ex
+    end
+  end
+  private :log_mapping_errors
+
+  # Create logger according to settings
+  def create_logger
+    logger_level  = settings['log.level'] || 'info'
+
+    # log everything to STDERR or specified logfile
+    logger        = Yell::Logger.new(:null)
+    logger.format = logger_format
+    logger.level  = logger_level
+
+    logger_destination = settings['log.file'] || 'STDERR'
+    # We intentionally repeat the logger_level
+    # on the adapter, so it will stay there if overall level
+    # is changed.
+    case logger_destination
+      when 'STDERR'
+        logger.adapter :stderr, level: logger_level, format: logger_format
+      when 'STDOUT'
+        logger.adapter :stdout, level: logger_level, format: logger_format
+      else
+        logger.adapter :file, logger_destination, level: logger_level, format: logger_format
+    end
+
+
+    # ADDITIONALLY log error and higher to....
+    if settings['log.error_file']
+      logger.adapter :file, settings['log.error_file'], :level => 'gte.error'
+    end
+
+    return logger
+  end
+  private :create_logger
+
+  def logger_format
+    format = settings['log.format'] || '%d %5L %m'
+    format = case format
+               when 'false' then
+                 false
+               when '' then
+                 nil
+               else
+                 format
+             end
+  end
+  private :logger_format
+end