traject 2.3.4 → 3.0.0.alpha.1

data/test/thread_pool_test.rb

@@ -0,0 +1,38 @@
+require 'test_helper'
+require 'rspec/mocks'
+
+# MOST of ThreadPool is not tested directly at this point.
+describe "Traject::ThreadPool" do
+  include ::RSpec::Mocks::ExampleMethods
+
+  # http://blog.plataformatec.com.br/2015/05/nobody-told-me-minitest-was-this-fun/
+  def before_setup
+    ::RSpec::Mocks.setup
+    super
+  end
+
+  def after_teardown
+    super
+    ::RSpec::Mocks.verify
+  ensure
+    ::RSpec::Mocks.teardown
+  end
+
+
+  describe "disable_concurrency!" do
+
+    it "disables concurrency" do
+      allow(Traject::ThreadPool).to receive(:concurrency_disabled?).and_return(true)
+
+      parent_thread_id = Thread.current.object_id
+
+      work_thread_id = Concurrent::AtomicFixnum.new
+
+      Traject::ThreadPool.new(10).maybe_in_thread_pool do
+        work_thread_id.update { Thread.current.object_id  }
+      end
+
+      assert_equal parent_thread_id, work_thread_id.value
+    end
+  end
+end

data/test/translation_map_test.rb

@@ -57,7 +57,7 @@ describe "TranslationMap" do
 
     it "raises on syntax error in yaml" do
       exception = assert_raises(Psych::SyntaxError) do
-        found = @cache.lookup("bad_yaml")
+        _found = @cache.lookup("bad_yaml")
       end
 
       assert  exception.message.include?("bad_yaml.yaml"), "exception message includes source file"
@@ -65,7 +65,7 @@ describe "TranslationMap" do
 
     it "raises on syntax error in ruby" do
       exception = assert_raises(SyntaxError) do
-        found = @cache.lookup("bad_ruby")
+        _found = @cache.lookup("bad_ruby")
       end
       assert  exception.message.include?("bad_ruby.rb"), "exception message includes source file"
     end
@@ -118,7 +118,7 @@ describe "TranslationMap" do
 
     assert_equal "DEFAULT LITERAL", map["not in the map"]
   end
-  
+
   it "respects __default__ __passthrough__" do
     map = Traject::TranslationMap.new("default_passthrough")
 

data/test/translation_maps/ruby_map.rb

@@ -2,7 +2,8 @@ foo = "bar"
 
 some_hash = {
   "key1" => "value1",
-  "array_key" => %w{one two three}
+  "array_key" => %w{one two three},
+  "key_to_be_overridden" => "value_from_ruby"
 }
 some_hash["also"] = "this"
 

data/test/translation_maps/yaml_map.yaml

@@ -1,7 +1,8 @@
 key1: value1
+key_to_be_overridden: value_from_yaml
 array_key:
   -one
   -two
   -three
 # comment
-other: yes
+other: yes

data/traject.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
   spec.version  = Traject::VERSION
   spec.authors  = ["Jonathan Rochkind", "Bill Dueber"]
   spec.email    = ["none@nowhere.org"]
-  spec.summary  = %q{Index MARC to Solr; or generally process source records to hash-like structures}
+  spec.summary  = %q{An easy to use, high-performance, flexible and extensible metadata transformation system, focused on library-archives-museums input, and indexing to Solr as output.}
   spec.homepage = "http://github.com/traject/traject"
   spec.license  = "MIT"
 
@@ -28,20 +28,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
+  spec.add_dependency "http", "~> 3.0" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
-
-  # If we're building the package under JRuby, add in the
-  # jruby-only gems and specify the platform.
-
-  if defined? JRUBY_VERSION
-    spec.platform = 'java'
-    spec.add_dependency "traject-marc4j_reader", "~> 1.0"
-  else
-    spec.platform = "ruby"
-  end
+  spec.add_dependency "nokogiri", "~> 1.0" # NokogiriIndexer
 
   spec.add_development_dependency "bundler", '~> 1.7'
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rspec-mocks", '~> 3.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 2.3.4
+  version: 3.0.0.alpha.1
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-06 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -115,6 +115,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: marc-fastxmlwriter
   requirement: !ruby/object:Gem::Requirement
@@ -129,6 +143,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +199,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
 description: 
 email:
 - none@nowhere.org
@@ -182,7 +224,9 @@ extra_rdoc_files:
 - doc/extending.md
 - doc/indexing_rules.md
 - doc/other_commands.md
+- doc/programmatic_use.md
 - doc/settings.md
+- doc/xml.md
 files:
 - ".gitignore"
 - ".travis.yml"
@@ -198,7 +242,9 @@ files:
 - doc/extending.md
 - doc/indexing_rules.md
 - doc/other_commands.md
+- doc/programmatic_use.md
 - doc/settings.md
+- doc/xml.md
 - index_bench/batch.dat
 - index_bench/common.rb
 - index_bench/index_bench.rb
@@ -216,12 +262,17 @@ files:
 - index_bench/translation_maps/umich/obsolete_cop.yaml
 - lib/tasks/load_maps.rake
 - lib/traject.rb
+- lib/traject/array_writer.rb
 - lib/traject/command_line.rb
 - lib/traject/csv_writer.rb
 - lib/traject/debug_writer.rb
 - lib/traject/delimited_writer.rb
+- lib/traject/experimental_nokogiri_streaming_reader.rb
+- lib/traject/hashie/indifferent_access_fix.rb
 - lib/traject/indexer.rb
 - lib/traject/indexer/context.rb
+- lib/traject/indexer/marc_indexer.rb
+- lib/traject/indexer/nokogiri_indexer.rb
 - lib/traject/indexer/settings.rb
 - lib/traject/indexer/step.rb
 - lib/traject/json_writer.rb
@@ -230,12 +281,16 @@ files:
 - lib/traject/macros/marc21.rb
 - lib/traject/macros/marc21_semantics.rb
 - lib/traject/macros/marc_format_classifier.rb
+- lib/traject/macros/nokogiri_macros.rb
+- lib/traject/macros/transformation.rb
 - lib/traject/marc_extractor.rb
 - lib/traject/marc_extractor_spec.rb
 - lib/traject/marc_reader.rb
 - lib/traject/mock_reader.rb
 - lib/traject/ndj_reader.rb
+- lib/traject/nokogiri_reader.rb
 - lib/traject/null_writer.rb
+- lib/traject/oai_pmh_nokogiri_reader.rb
 - lib/traject/qualified_const_get.rb
 - lib/traject/solr_json_writer.rb
 - lib/traject/thread_pool.rb
@@ -251,8 +306,10 @@ files:
 - lib/translation_maps/marc_languages.yaml
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
+- test/experimental_nokogiri_streaming_reader_test.rb
 - test/indexer/context_test.rb
 - test/indexer/each_record_test.rb
+- test/indexer/error_handler_test.rb
 - test/indexer/load_config_file_test.rb
 - test/indexer/macros/macros_marc21_semantics_test.rb
 - test/indexer/macros/marc21/extract_all_marc_values_test.rb
@@ -260,7 +317,11 @@ files:
 - test/indexer/macros/marc21/serialize_marc_test.rb
 - test/indexer/macros/marc21/trim_punctuation_test.rb
 - test/indexer/macros/to_field_test.rb
+- test/indexer/macros/transformation_test.rb
 - test/indexer/map_record_test.rb
+- test/indexer/nokogiri_indexer_test.rb
+- test/indexer/process_record_test.rb
+- test/indexer/process_with_test.rb
 - test/indexer/read_write_test.rb
 - test/indexer/settings_test.rb
 - test/indexer/to_field_test.rb
@@ -268,6 +329,8 @@ files:
 - test/marc_extractor_test.rb
 - test/marc_format_classifier_test.rb
 - test/marc_reader_test.rb
+- test/nokogiri_reader_test.rb
+- test/oai_pmh_nokogiri_reader_test.rb
 - test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc
@@ -289,15 +352,22 @@ files:
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc
+- test/test_support/namespace-test.xml
 - test/test_support/nature.marc
+- test/test_support/nokogiri_demo_config.rb
+- test/test_support/oai-pmh-one-record-2.xml
+- test/test_support/oai-pmh-one-record-first.xml
 - test/test_support/one-marc8.mrc
 - test/test_support/online_only.marc
 - test/test_support/packed_041a_lang.marc
+- test/test_support/sample-oai-no-namespace.xml
+- test/test_support/sample-oai-pmh.xml
 - test/test_support/test_data.utf8.json
 - test/test_support/test_data.utf8.marc.xml
 - test/test_support/test_data.utf8.mrc
 - test/test_support/test_data.utf8.mrc.gz
 - test/test_support/the_business_ren.marc
+- test/thread_pool_test.rb
 - test/translation_map_test.rb
 - test/translation_maps/bad_ruby.rb
 - test/translation_maps/bad_yaml.yaml
@@ -326,20 +396,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
-summary: Index MARC to Solr; or generally process source records to hash-like structures
+summary: An easy to use, high-performance, flexible and extensible metadata transformation
+  system, focused on library-archives-museums input, and indexing to Solr as output.
 test_files:
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
+- test/experimental_nokogiri_streaming_reader_test.rb
 - test/indexer/context_test.rb
 - test/indexer/each_record_test.rb
+- test/indexer/error_handler_test.rb
 - test/indexer/load_config_file_test.rb
 - test/indexer/macros/macros_marc21_semantics_test.rb
 - test/indexer/macros/marc21/extract_all_marc_values_test.rb
@@ -347,7 +420,11 @@ test_files:
 - test/indexer/macros/marc21/serialize_marc_test.rb
 - test/indexer/macros/marc21/trim_punctuation_test.rb
 - test/indexer/macros/to_field_test.rb
+- test/indexer/macros/transformation_test.rb
 - test/indexer/map_record_test.rb
+- test/indexer/nokogiri_indexer_test.rb
+- test/indexer/process_record_test.rb
+- test/indexer/process_with_test.rb
 - test/indexer/read_write_test.rb
 - test/indexer/settings_test.rb
 - test/indexer/to_field_test.rb
@@ -355,6 +432,8 @@ test_files:
 - test/marc_extractor_test.rb
 - test/marc_format_classifier_test.rb
 - test/marc_reader_test.rb
+- test/nokogiri_reader_test.rb
+- test/oai_pmh_nokogiri_reader_test.rb
 - test/solr_json_writer_test.rb
 - test/test_helper.rb
 - test/test_support/245_no_ab.marc
@@ -376,15 +455,22 @@ test_files:
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc
+- test/test_support/namespace-test.xml
 - test/test_support/nature.marc
+- test/test_support/nokogiri_demo_config.rb
+- test/test_support/oai-pmh-one-record-2.xml
+- test/test_support/oai-pmh-one-record-first.xml
 - test/test_support/one-marc8.mrc
 - test/test_support/online_only.marc
 - test/test_support/packed_041a_lang.marc
+- test/test_support/sample-oai-no-namespace.xml
+- test/test_support/sample-oai-pmh.xml
 - test/test_support/test_data.utf8.json
 - test/test_support/test_data.utf8.marc.xml
 - test/test_support/test_data.utf8.mrc
 - test/test_support/test_data.utf8.mrc.gz
 - test/test_support/the_business_ren.marc
+- test/thread_pool_test.rb
 - test/translation_map_test.rb
 - test/translation_maps/bad_ruby.rb
 - test/translation_maps/bad_yaml.yaml