traject 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a69bdb470c759d08d117f910ec1c6b16cbe93dbb3e955b653a07e3c03efec68
-  data.tar.gz: 68016ddf17fe29348248e5f26ff5a813c1485c37a0ac014faa69803be182f7b4
+  metadata.gz: 1700077d5c2d3c667fc9520b659c3ca986b8ab34aee233f62bd7f73fdef91977
+  data.tar.gz: 736b217f209ed08faba9c1d20c006b29586aa3ebdf088a89e37f5f3b7400de06
 SHA512:
-  metadata.gz: 8921a5d2349025291f28c7d39ebec44c0c0ebdc11a489e1d4c22913c04c840cf40163c576b14432794f1d663b9b81eb1d777b87d38af2b429f4b6ac98982ad48
-  data.tar.gz: 8bdb95f98baa11ee60c74a7a4bbab06c44fb5da466138d098b05e5ab3fddc28a1fe8926044be5350e5e0b8a91ccd2d6e2e2ecf78424cfc0430464dd0ccf79c66
+  metadata.gz: 21877d6cd5b03f7ffbbac316a6d58a3bc65b534cb7457e57d39ba470ad49d99c8677e5e6ede25c650bba5ac3f0b22f9b348ebabb36ac4047433eb8a76379ef1d
+  data.tar.gz: 4ec1938d2d7b60a61ebde4e9c4e763e511c2896788b56b38ad6f22615dffb57449e29c9ef40e261952e125b90f5fe491fa447b077c7dcb1c55f57d6ef603fd5b

data/CHANGES.md

@@ -8,6 +8,16 @@
 
 *
 
+## 3.3.0
+
+* `Traject::Macros::Marc21Semantics.publication_date` now gets date from 264 before 260. https://github.com/traject/traject/pull/233
+
+* Allow hashie 4.x in gemspec https://github.com/traject/traject/pull/234
+
+* Allow `http` gem 4.x versions. https://github.com/traject/traject/pull/236
+
+* Can now call class-level Indexer.configure multiple times https://github.com/sciencehistory/scihist_digicoll/pull/525
+
 ## 3.2.0
 
 * NokogiriReader has a "nokogiri.strict_mode" setting. Set to true or string 'true' to ask Nokogori to parse in strict mode, so it will immediately raise on ill-formed XML, instead of nokogiri's default to do what it can with it. https://github.com/traject/traject/pull/226

data/lib/traject/indexer.rb

@@ -190,7 +190,7 @@ class Traject::Indexer
     instance_eval(&block)
   end
 
-  ## Class level configure block accepted too, and applied at instantiation
+  ## Class level configure block(s) accepted too, and applied at instantiation
   #  before instance-level configuration.
   #
   #  EXPERIMENTAL, implementation may change in ways that effect some uses.
@@ -199,8 +199,14 @@ class Traject::Indexer
   #  Note that settings set by 'provide' in subclass can not really be overridden
   #  by 'provide' in a next level subclass. Use self.default_settings instead, with
   #  call to super.
+  #
+  #  You can call this .configure multiple times, blocks are added to a list, and
+  #  will be used to initialize an instance in order.
+  #
+  #  The main downside of this workaround implementation is performance, even though
+  #  defined at load-time on class level, blocks are all executed on every instantiation.
   def self.configure(&block)
-    @class_configure_block = block
+    (@class_configure_blocks ||= []) << block
   end
 
   def self.apply_class_configure_block(instance)
@@ -208,8 +214,10 @@ class Traject::Indexer
     if self.superclass.respond_to?(:apply_class_configure_block)
       self.superclass.apply_class_configure_block(instance)
     end
-    if @class_configure_block
-      instance.configure(&@class_configure_block)
+    if @class_configure_blocks && !@class_configure_blocks.empty?
+      @class_configure_blocks.each do |block|
+        instance.configure(&block)
+      end
     end
   end
 

data/lib/traject/macros/marc21_semantics.rb

@@ -26,19 +26,19 @@ module Traject::Macros
         accumulator.concat list.uniq if list
       end
     end
-   
+
     # If a num begins with a known OCLC prefix, return it without the prefix.
     # otherwise nil.
     #
-    # Allow (OCoLC) and/or ocn/ocm/on 
-    
+    # Allow (OCoLC) and/or ocn/ocm/on
+
     OCLCPAT = /
       \A\s*
       (?:(?:\(OCoLC\)) |
          (?:\(OCoLC\))?(?:(?:ocm)|(?:ocn)|(?:on))
          )(\d+)
          /x
-         
+
     def self.oclcnum_extract(num)
       if m = OCLCPAT.match(num)
         return m[1]
@@ -364,13 +364,16 @@ module Traject::Macros
           end
         end
       end
-      # Okay, nothing from 008, try 260
+      # Okay, nothing from 008, first try 264, then try 260
       if found_date.nil?
+        v264c = MarcExtractor.cached("264c", :separator => nil).extract(record).first
         v260c = MarcExtractor.cached("260c", :separator => nil).extract(record).first
         # just try to take the first four digits out of there, we're not going to try
         # anything crazy.
-        if m = /(\d{4})/.match(v260c)
+        if m = /(\d{4})/.match(v264c)
           found_date = m[1].to_i
+        elsif m = /(\d{4})/.match(v260c)
+            found_date = m[1].to_i
         end
       end
 
@@ -519,11 +522,11 @@ module Traject::Macros
 
     # Extracts LCSH-carrying fields, and formatting them
     # as a pre-coordinated LCSH string, for instance suitable for including
-    # in a facet. 
+    # in a facet.
     #
     # You can supply your own list of fields as a spec, but for significant
     # customization you probably just want to write your own method in
-    # terms of the Marc21Semantics.assemble_lcsh method. 
+    # terms of the Marc21Semantics.assemble_lcsh method.
     def marc_lcsh_formatted(options = {})
       spec            = options[:spec] || "600:610:611:630:648:650:651:654:662"
       subd_separator  = options[:subdivison_separator] || " — "
@@ -540,17 +543,17 @@ module Traject::Macros
     end
 
     # Takes a MARC::Field and formats it into a pre-coordinated LCSH string
-    # with subdivision seperators in the right place. 
+    # with subdivision seperators in the right place.
     #
     # For 600 fields especially, need to not just join with subdivision seperator
     # to take acount of $a$d$t -- for other fields, might be able to just
-    # join subfields, not sure. 
+    # join subfields, not sure.
     #
     # WILL strip trailing period from generated string, contrary to some LCSH practice.
     # Our data is inconsistent on whether it has period or not, this was
-    # the easiest way to standardize. 
+    # the easiest way to standardize.
     #
-    # Default subdivision seperator is em-dash with spaces, set to '--' if you want. 
+    # Default subdivision seperator is em-dash with spaces, set to '--' if you want.
     #
     # Cite: "Dash (-) that precedes a subdivision in an extended 600 subject heading
     # is not carried in the MARC record. It may be system generated as a display constant

data/lib/traject/oai_pmh_nokogiri_reader.rb

@@ -115,9 +115,15 @@ module Traject
     # @returns [HTTP::Client] from http.rb gem
     def http_client
       @http_client ||= begin
-        # timeout setting on http.rb seems to be a mess.
-        # https://github.com/httprb/http/issues/488
-        client = HTTP.timeout(:global, write: timeout / 3, connect: timeout / 3, read: timeout / 3)
+        client = nil
+
+        if HTTP::VERSION.split(".").first.to_i > 3
+          client = HTTP.timeout(timeout)
+        else
+          # timeout setting on http.rb 3.x are a bit of a mess.
+          # https://github.com/httprb/http/issues/488
+          client = HTTP.timeout(:global, write: timeout / 3, connect: timeout / 3, read: timeout / 3)
+        end
 
         if settings["oai_pmh.try_gzip"]
           client = client.use(:auto_inflate).headers("accept-encoding" => "gzip;q=1.0, identity;q=0.5")

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.2.0"
+  VERSION = "3.3.0"
 end

data/test/delimited_writer_test.rb

@@ -24,40 +24,40 @@ describe "Delimited/CSV Writers" do
 
     it "creates a dw with defaults" do
       dw = Traject::DelimitedWriter.new(@settings)
-      dw.delimiter.must_equal "\t"
-      dw.internal_delimiter.must_equal '|'
-      dw.edelim.must_equal ' '
-      dw.eidelim.must_equal '\\|'
+      assert_equal dw.delimiter, "\t"
+      assert_equal dw.internal_delimiter, '|'
+      assert_equal dw.edelim, ' '
+      assert_equal dw.eidelim, '\\|'
     end
 
     it "respects different delimiter" do
       @settings['delimited_writer.delimiter'] = '^'
       dw                                      = Traject::DelimitedWriter.new(@settings)
-      dw.delimiter.must_equal '^'
-      dw.edelim.must_equal '\\^'
-      dw.internal_delimiter.must_equal '|'
+      assert_equal dw.delimiter, '^'
+      assert_equal dw.edelim, '\\^'
+      assert_equal dw.internal_delimiter, '|'
     end
 
     it "outputs a header if asked to" do
       Traject::DelimitedWriter.new(@settings)
-      @out.string.chomp.must_equal %w[four one two].join("\t")
+      assert_equal @out.string.chomp, %w[four one two].join("\t")
     end
 
     it "doesn't output a header if asked not to" do
       @settings['delimited_writer.header'] = 'false'
       Traject::DelimitedWriter.new(@settings)
-      @out.string.must_be_empty
+      assert_empty @out.string
     end
 
     it "deals with multiple values" do
       dw = Traject::DelimitedWriter.new(@settings)
       dw.put @context
-      @out.string.split("\n").last.must_equal ['four', 'one', 'two1|two2'].join(dw.delimiter)
+      assert_equal @out.string.split("\n").last, ['four', 'one', 'two1|two2'].join(dw.delimiter)
     end
 
     it "bails if delimited_writer.fields isn't set" do
       @settings.delete 'delimited_writer.fields'
-      proc { Traject::DelimitedWriter.new(@settings) }.must_raise(ArgumentError)
+      assert_raises(ArgumentError)  { Traject::DelimitedWriter.new(@settings) }
     end
 
   end
@@ -65,18 +65,18 @@ describe "Delimited/CSV Writers" do
   describe "Traject::CSVWriter" do
     it "unsets the delimiter" do
       cw = Traject::CSVWriter.new(@settings)
-      cw.delimiter.must_be_nil
+      assert_nil cw.delimiter
     end
 
     it "writes the header" do
       Traject::CSVWriter.new(@settings)
-      @out.string.chomp.must_equal 'four,one,two'
+      assert_equal @out.string.chomp, 'four,one,two'
     end
 
     it "uses the internal delimiter" do
       cw = Traject::CSVWriter.new(@settings)
       cw.put @context
-      @out.string.split("\n").last.must_equal ['four', 'one', 'two1|two2'].join(',')
+      assert_equal @out.string.split("\n").last, ['four', 'one', 'two1|two2'].join(',')
     end
 
     it "produces complex output" do
@@ -97,8 +97,6 @@ describe "Delimited/CSV Writers" do
       traject_csvwriter_output = @out.string.split("\n").last.chomp
 
       assert_equal(csv_output, traject_csvwriter_output)
-
     end
-
   end
 end

data/test/indexer/class_level_configuration_test.rb

@@ -25,6 +25,7 @@ describe "Class-level configuration of Indexer sub-class" do
     end
   end
 
+
   before do
     @indexer = TestIndexerSubclass.new
   end
@@ -47,6 +48,28 @@ describe "Class-level configuration of Indexer sub-class" do
     assert_equal ['from-instance-config'], result["instance_field"]
   end
 
+  describe "multiple class-level configure" do
+    class MultipleConfigureIndexer < Traject::Indexer
+      configure do
+        to_field "field", literal("value")
+      end
+      configure do
+        to_field "field", literal("value from second configure")
+        to_field "second_call", literal("value from second configure")
+      end
+    end
+
+    before do
+      @indexer = MultipleConfigureIndexer.new
+    end
+
+    it "lets you call class-level configure multiple times and aggregates" do
+      result = @indexer.map_record(Object.new)
+      assert_equal ['value', 'value from second configure'], result['field']
+      assert_equal ['value from second configure'], result['second_call']
+    end
+  end
+
   describe "with multi-level subclass" do
     class TestIndexerSubclassSubclass < TestIndexerSubclass
       configure do

data/test/indexer/macros/macros_marc21_semantics_test.rb

@@ -197,6 +197,10 @@ describe "Traject::Macros::Marc21Semantics" do
       # we take the first date. And need to deal with the u.
       assert_equal 1845, Marc21Semantics.publication_date(@record)
     end
+    it "resorts to 264c" do
+      @record = MARC::Reader.new(support_file_path  "date_resort_to_264.marc").to_a.first
+      assert_equal 2015, Marc21Semantics.publication_date(@record)
+    end
     it "resorts to 260c" do
       @record = MARC::Reader.new(support_file_path  "date_resort_to_260.marc").to_a.first
       assert_equal 1980, Marc21Semantics.publication_date(@record)

data/test/test_support/date_resort_to_264.marc

@@ -0,0 +1 @@
+01180aam a2200337 a 4500001001000000008004100010015001900051016001800070020002500088020002200113040005900135043001200194050002400206082001400230100003500244245006400279260003800343264003800381264001100419300003600430336002100466336002800487337002500515338002300540504006700563651004000630651005000670651004300720651005600763035002300819a11417842130723t20uu20uuenkb     b    001 0 eng d  aGBB3854302bnb7 a0164999372Uk  a9781849043427 (pbk.)  a1849043426 (pbk.)  aUKMGBcUKMGBdOCLCOdYDXCPdOCLCOdZWZdOCLCOdCaONFJC  aa-ii--- 4aDS485.K25bS64 201504a954.62231 aSnedden, Christopher,eauthor.10aUnderstanding Kashmir and Kashmiris /cChristopher Snedden. 1aLondon :bHurst & Company,c2014. 1aLondon :bHurst & Company,c2015. 4c©2015  axix, 372 pages :bmaps ;c22 cm  atext2rdacontent  astill image2rdacontent  aunmediated2rdamedia  avolume2rdacarrier  aIncludes bibliographical references (pages 331-355) and index. 0aJammu and Kashmir (India)xHistory. 0aJammu and Kashmir (India)xForeign relations. 0aJammu and Kashmir (India)vBoundaries. 0aJammu and Kashmir (India)xPolitics and government.  a(OCoLC-M)858826393

data/traject.gemspec

@@ -24,12 +24,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency "concurrent-ruby", ">= 0.8.0"
   spec.add_dependency "marc", "~> 1.0"
 
-  spec.add_dependency "hashie", "~> 3.1" # used for Indexer#settings
+  spec.add_dependency "hashie", ">= 3.1", "< 5" # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0" # command line parsing
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
-  spec.add_dependency "http", "~> 3.0" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
+  spec.add_dependency "http", ">= 3.0", "< 5" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-10 00:00:00.000000000 Z
+date: 2019-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -43,16 +43,22 @@ dependencies:
   name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -119,16 +125,22 @@ dependencies:
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: marc-fastxmlwriter
   requirement: !ruby/object:Gem::Requirement
@@ -330,6 +342,7 @@ files:
 - test/test_support/bad_subfield_code.marc
 - test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
+- test/test_support/date_resort_to_264.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
@@ -434,6 +447,7 @@ test_files:
 - test/test_support/bad_subfield_code.marc
 - test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
+- test/test_support/date_resort_to_264.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb