traject 3.6.0 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e47e6648ed9fc963d18e10c9be48a30273147c4920cb4b7e448d078fd2398ac
-  data.tar.gz: efa549ebcbd87e599b56b955b4bd26422dfe7de67697aed6b39cb421c3b80677
+  metadata.gz: 7ffc677e0ebb13e01b852a1d59ddfdd3cd9906142520e0c296f69ebb0eeb7429
+  data.tar.gz: 61b0e966f6ecd4d27e757e4cfc1057c72ac6deca5ad119c78ff883c246744814
 SHA512:
-  metadata.gz: 6acdd2b8cfc888b221a1f19cd5197127006be81d0525169d531fc9bf43fe02cc9ec87401e6b2442c57ff0cd483d9884504ac75be92e3718cbbc49208dc97024f
-  data.tar.gz: 30abefa7af9e1c170ae8570aa59b6c571a9acc1eb7b0abf6efd64d97550b678c21c72d76a8156ef3844ab01154111fd3747f96d6346ee8a8d76e747b2cf92e1f
+  metadata.gz: 8240b450b27df011c2ff998c24c612f44bdd21a2fde5fbab996ffe509f3fc45cec7a8a8947e385a35d06f5bd8ed19732287e8f2d3dab682cc6295f7320f8dfab
+  data.tar.gz: f5dbcb44edb8d37a4e74cd1255aa1b05b638913337577c92d4fa276150c023b9e29823cc2b20ef050b33879ec63d76194d0202697d7c858a7aacd3d08241dcce

data/CHANGES.md

@@ -1,11 +1,15 @@
 # Changes
 
-## Next
+## NEXT
 
 *
 
 *
 
+## 3.7.0
+
+* Add two new transformation macros, `Traject::Macros::Transformation.delete_if` and `Traject::Macros::Transformations.select`.
+
 ## 3.6.0
 
 * Tiny backward compat changes for ruby 3.0 compat. https://github.com/traject/traject/pull/263

data/README.md

@@ -177,6 +177,11 @@ TranslationMap use above is just one example of a transformation macro, that tra
 * `split(" ")`: take values and split them, possibly result in multiple values.
 * `transform(proc)`: transform each existing macro using a proc, kind of like `map`.
    eg `to_field "something", extract_xml("//author"), transform( ->(author) { "#{author.last}, #{author.first}" })
+* `delete_if(["a", "b"])`: remove a value from accumulated values if it is included in the passed in argumet.
+    * Can also take a string, proc or regex as an argument. See [tests](test/indexer/macros/transformation_test.rb) for full functionality.
+* `select(proc)`: selects (keeps) values from accumulated values if proc evaluates to true for specifc value.
+    * Can also take a arrays, sets and regex as an argument. See [tests](test/indexer/macros/transformation_test.rb) for full functionality.
+
 
 You can add on as many transformation macros as you want, they will be applied to output in order.
 

data/lib/traject/macros/marc21_semantics.rb

@@ -327,10 +327,14 @@ module Traject::Macros
       if field008 && field008.length >= 11
         date_type = field008.slice(6)
         date1_str = field008.slice(7,4)
-        date2_str = field008.slice(11, 4) if field008.length > 15
+        if field008.length > 15
+          date2_str = field008.slice(11, 4)
+        else
+          date2_str = date1_str
+        end
 
-        # for date_type q=questionable, we have a range.
-        if (date_type == 'q')
+        # for date_type q=questionable, we expect to have a range.
+        if date_type == 'q' and date1_str != date2_str
           # make unknown digits at the beginning or end of range,
           date1 = date1_str.sub("u", "0").to_i
           date2 = date2_str.sub("u", "9").to_i

data/lib/traject/macros/transformation.rb

@@ -157,6 +157,36 @@ module Traject
           acc.collect! { |v| v.gsub(pattern, replace) }
         end
       end
+
+      # Run ruby `delete_if` on the accumulator for values that include or are equal to arg.
+      # It will also accept an array, set, regex pattern, proc or lambda as an arugment.
+      #
+      # @example
+      #   to_field "creator_facet", extract_marc("100abcdq"), delete_if(/foo/)
+      def delete_if(arg)
+        p = if arg.respond_to? :include?
+              proc { |v| arg.include?(v) }
+            else
+              proc { |v| arg === v }
+            end
+
+        ->(_, acc) { acc.delete_if(&p) }
+      end
+
+      # Run ruby `select!` on the accumulator for values that include or are equal to arg.
+      # It accepts an array, set, regex pattern, proc or lambda as an arugument.
+      #
+      # @example
+      #   to_field "creator_facet", extract_marc("100abcdq"), select(->(v) { v != "foo" })
+      def select(arg)
+        p = if arg.respond_to? :include?
+              proc { |v| arg.include?(v) }
+            else
+              proc { |v| arg === v }
+            end
+
+        ->(_, acc) { acc.select!(&p) }
+      end
     end
   end
 end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.6.0"
+  VERSION = "3.7.0"
 end

data/test/indexer/macros/macros_marc21_semantics_test.rb

@@ -209,6 +209,10 @@ describe "Traject::Macros::Marc21Semantics" do
       @record = MARC::Reader.new(support_file_path  "date_type_r_missing_date2.marc").to_a.first
       assert_equal 1957, Marc21Semantics.publication_date(@record)
     end
+    it "provides a fallback for a missing second date" do
+      @record = MARC::Reader.new(support_file_path  "missing-second-date.marc").to_a.first
+      assert_equal 1678, Marc21Semantics.publication_date(@record)
+    end
 
     it "works correctly with date type 'q'" do
       val = @record['008'].value

data/test/indexer/macros/transformation_test.rb

@@ -174,4 +174,114 @@ describe "Traject::Macros::Transformation" do
     end
   end
 
+  describe "delete_if" do
+
+    describe "argument is an Array" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "filters out selected values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "filters out selected values from accumulatd values" do
+        arg = ->(v) { v == "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
+  describe "select" do
+
+    describe "argument is an Array" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "selects a subset of values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "selects a subset of values from accumulatd values" do
+        arg = ->(v) { v != "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
 end

data/test/test_support/missing-second-date.marc

@@ -0,0 +1 @@
+01351nem a2200313 a 4500001001100000001001100011008004100022034001300063035002200076043003000098080001200128100006100140245011600201255002900317260005900346300003500405500004500440500013600485500003800621500005000659530004400709651006000753700003800813710010200851730000900953856005300962907001401015940000801029.b20028118.b6928510x170714q1678    fr ||||   |  |||| ||fre|c1 aab10000  a(OCoLC)1120596466  ae-sp---be-spcce2catmarc  a(084.3)1 aBeaulieu, Sébastien de Pontault,csieur de,d1613-1674.10aPlan de la ville de Puiçerdah[Document cartogràfic] :bpris en 1678 /c[Beaulieu] ; DR f. [Des Roches fecit]  aEscala [1:10 000 aprox.]  a[A Paris :bpar le Chevalier de Beaulieu,cpost. 1678]  a1 mapa :bgravat;c28 x 32 cm.  aEscala gràfica: Eschelle de 150 toises.  aPertany a l'obra "Les plans et profils des principales villes et lieux considerables de la Principauté de Catalogne", de Beaulieu.  aPeu d'impremta de l'obra general.  aMapa emmarcat en una orla amb motius florals.  aTambé disponible la versió en línia. 4aPuigcerdà (Catalunya)xMapesxObres anteriors al 1800.1 aDes Roches, Jean Baptiste Hamont.2 aCol·lecció de mapes antics de Martí Gelabertó (Universitat Autònoma de Barcelona)5ES-BaUAB.0 aDDD.41zAccés lliureuhttps://ddd.uab.cat/record/180313  ab20028118  aUAB

data/traject.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "concurrent-ruby", ">= 0.8.0"
   spec.add_dependency "marc", "~> 1.0"
 
-  spec.add_dependency "hashie", ">= 3.1", "< 5" # used for Indexer#settings
+  spec.add_dependency "hashie", ">= 3.1", "< 6" # used for Indexer#settings
   spec.add_dependency "slop", "~> 4.0" # command line parsing
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.7.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2022-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -48,7 +48,7 @@ dependencies:
         version: '3.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
         version: '3.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -350,6 +350,7 @@ files:
 - test/test_support/manufacturing_consent.marc
 - test/test_support/manuscript_online_thesis.marc
 - test/test_support/microform_online_conference.marc
+- test/test_support/missing-second-date.marc
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc
@@ -401,7 +402,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
@@ -455,6 +456,7 @@ test_files:
 - test/test_support/manufacturing_consent.marc
 - test/test_support/manuscript_online_thesis.marc
 - test/test_support/microform_online_conference.marc
+- test/test_support/missing-second-date.marc
 - test/test_support/multi_era.marc
 - test/test_support/multi_geo.marc
 - test/test_support/musical_cage.marc