omelette 0.0.1a

data/lib/omelette/util.rb

@@ -0,0 +1,146 @@
+require 'rest-client'
+
+module Omelette
+  module Util
+
+    def exception_to_log_message(e)
+      indent = '    '
+
+      msg = indent + 'Exception: ' + e.class.name + ': ' + e.message + "\n"
+      msg += indent + e.backtrace.first + "\n"
+
+      if (e.respond_to?(:getRootCause) && e.getRootCause && e != e.getRootCause)
+        caused_by = e.getRootCause
+        msg       += indent + "Caused by\n"
+        msg       += indent + caused_by.class.name + ': ' + caused_by.message + "\n"
+        msg       += indent + caused_by.backtrace.first + "\n"
+      end
+
+      return msg
+    end
+    module_function :exception_to_log_message
+
+    # From ruby #caller method, you get an array. Pass one line
+    # of the array here,  get just file and line number out.
+    def extract_caller_location(str)
+      str.split(':in `').first
+    end
+    module_function :extract_caller_location
+
+    # Provide a config source file path, and an exception.
+    #
+    # Returns the line number from the first line in the stack
+    # trace of the exception that matches your file path.
+    # of the first line in the backtrace matching that file_path.
+    #
+    # Returns `nil` if no suitable backtrace line can be found.
+    #
+    # Has special logic to try and grep the info out of a SyntaxError, bah.
+    def backtrace_lineno_for_config(file_path, exception)
+      # For a SyntaxError, we really need to grep it from the
+      # exception message, it really appears to be nowhere else. Ugh.
+      if exception.kind_of? SyntaxError
+        if m = /:(\d+):/.match(exception.message)
+          return m[1].to_i
+        end
+      end
+
+      # Otherwise we try to fish it out of the backtrace, first
+      # line matching the config file path.
+
+      # exception.backtrace_locations exists in MRI 2.1+, which makes
+      # our task a lot easier. But not yet in JRuby 1.7.x, so we got to
+      # handle the old way of having to parse the strings in backtrace too.
+      if (exception.respond_to?(:backtrace_locations) &&
+          exception.backtrace_locations &&
+          exception.backtrace_locations.length > 0)
+        location = exception.backtrace_locations.find do |bt|
+          bt.path == file_path
+        end
+        return location ? location.lineno : nil
+      else # have to parse string backtrace
+        exception.backtrace.each do |line|
+          if line.start_with?(file_path)
+            if m = /\A.*\:(\d+)\:in/.match(line)
+              return m[1].to_i
+              break
+            end
+          end
+        end
+        # if we got here, we have nothing
+        return nil
+      end
+    end
+    module_function :backtrace_lineno_for_config
+
+    # Extract just the part of the backtrace that is "below"
+    # the config file mentioned. If we can't find the config file
+    # in the stack trace, we might return empty array.
+    #
+    # If the ruby supports Exception#backtrace_locations, the
+    # returned array will actually be of Thread::Backtrace::Location elements.
+    def backtrace_from_config(file_path, exception)
+      filtered_trace = []
+      found          = false
+
+      # MRI 2.1+ has exception.backtrace_locations which makes
+      # this a lot easier, but JRuby 1.7.x doesn't yet, so we
+      # need to do it both ways.
+      if (exception.respond_to?(:backtrace_locations) &&
+          exception.backtrace_locations &&
+          exception.backtrace_locations.length > 0)
+
+        exception.backtrace_locations.each do |location|
+          filtered_trace << location
+          (found=true and break) if location.path == file_path
+        end
+      else
+        filtered_trace = []
+        exception.backtrace.each do |line|
+          filtered_trace << line
+          (found=true and break) if line.start_with?(file_path)
+        end
+      end
+
+      return found ? filtered_trace : []
+    end
+    module_function :backtrace_from_config
+
+    # Ruby stdlib queue lacks a 'drain' function, we write one.
+    #
+    # Removes everything currently in the ruby stdlib queue, and returns
+    # it an array.  Should be concurrent-safe, but queue may still have
+    # some things in it after drain, if there are concurrent writers.
+    def drain_queue(queue)
+      result = []
+
+      queue_size = queue.size
+      begin
+        queue_size.times do
+          result << queue.deq(:raise_if_empty)
+        end
+      rescue ThreadError
+        # Need do nothing, queue was concurrently popped, no biggie
+      end
+
+      return result
+    end
+    module_function :drain_queue
+
+    def build_elements_map(api_root)
+      result = RestClient.get "#{api_root}/element_sets"
+      element_sets = JSON.parse(result.body)
+      element_sets_map = element_sets.map { |s| [s['id'], s['name']] }.to_h
+      elements_map = element_sets.map { |s| [s['name'], {}] }.to_h
+
+      result = RestClient.get "#{api_root}/elements"
+      elements = JSON.parse(result.body)
+      elements.each do |element|
+        element_set_name = element_sets_map[element['element_set']['id']]
+        elements_map[element_set_name][element['name']] = element['id']
+      end
+      return elements_map
+    end
+    module_function :build_elements_map
+  end
+end

data/lib/omelette/version.rb

@@ -0,0 +1,3 @@
+module Omelette
+  VERSION = '0.0.1a'
+end

data/lib/omelette/xml_reader.rb

@@ -0,0 +1,27 @@
+class Omelette::XmlReader
+  attr_reader :settings, :folder
+
+  def initialize(settings, file_paths)
+    @settings = Omelette::Importer::Settings.new settings
+    @file_paths = file_paths
+  end
+
+  def logger
+    @logger ||= (@settings[:logger] || Yell.new(STDERR, :level => "gt.fatal"))
+  end
+
+  def each
+    return enum_for(:each) unless block_given?
+
+
+      @file_paths.each do |file|
+      begin
+      xml_doc = Nokogiri::XML(File.open(file))
+      xml_doc.remove_namespaces! if settings['remove_xml_namespaces'].to_s == 'true'
+      yield xml_doc, File.basename(file)
+      rescue => ex
+        self.logger.error "Problem processing file #{file}: #{ex.message}"
+      end
+    end
+  end
+end

data/omelette.gemspec

@@ -0,0 +1,36 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "omelette/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "omelette"
+  spec.version       = Omelette::VERSION
+  spec.authors       = ["Dazhi Jiao"]
+  spec.email         = ["dazhi.jiao@gmail.com"]
+
+  spec.summary       = "A tool that imports data into Omeka using the Omeka API"
+  spec.description   = "Omelette is a Ruby gem that uses the Omeka API to import data. " \
+                       "It has a DSL and can be used or extended to process almost any kinds of data." \
+                       "It was inspired by the traject gem."
+  spec.homepage      = "http://github.com/jiaola/omelette"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "thor", "~> 0.20"
+  spec.add_runtime_dependency "nokogiri", "~> 1.8"
+  spec.add_runtime_dependency "rest-client", "~> 2.0"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.0"
+  spec.add_runtime_dependency "hashie", "~> 3.5"
+  spec.add_runtime_dependency "yell", "~> 2.0"
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "webmock", "~> 3.0"
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: omelette
+version: !ruby/object:Gem::Version
+  version: 0.0.1a
+platform: ruby
+authors:
+- Dazhi Jiao
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: yell
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Omelette is a Ruby gem that uses the Omeka API to import data. It has
+  a DSL and can be used or extended to process almost any kinds of data.It was inspired
+  by the traject gem.
+email:
+- dazhi.jiao@gmail.com
+executables:
+- omelette
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/omelette
+- lib/omelette.rb
+- lib/omelette/command_line.rb
+- lib/omelette/importer.rb
+- lib/omelette/importer/context.rb
+- lib/omelette/importer/errors.rb
+- lib/omelette/importer/settings.rb
+- lib/omelette/importer/steps.rb
+- lib/omelette/macros/xpath.rb
+- lib/omelette/null_writer.rb
+- lib/omelette/omeka_json_writer.rb
+- lib/omelette/thread_pool.rb
+- lib/omelette/util.rb
+- lib/omelette/version.rb
+- lib/omelette/xml_reader.rb
+- omelette.gemspec
+homepage: http://github.com/jiaola/omelette
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: A tool that imports data into Omeka using the Omeka API
+test_files: []