traject 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1700077d5c2d3c667fc9520b659c3ca986b8ab34aee233f62bd7f73fdef91977
-  data.tar.gz: 736b217f209ed08faba9c1d20c006b29586aa3ebdf088a89e37f5f3b7400de06
+  metadata.gz: c30572335810dc620f9a169df6f8f374512d3c472ea34bc03068106959fd1463
+  data.tar.gz: 3181c37e41e80416487d730e1983bc647daf480a3a308db30e294c7587adc644
 SHA512:
-  metadata.gz: 21877d6cd5b03f7ffbbac316a6d58a3bc65b534cb7457e57d39ba470ad49d99c8677e5e6ede25c650bba5ac3f0b22f9b348ebabb36ac4047433eb8a76379ef1d
-  data.tar.gz: 4ec1938d2d7b60a61ebde4e9c4e763e511c2896788b56b38ad6f22615dffb57449e29c9ef40e261952e125b90f5fe491fa447b077c7dcb1c55f57d6ef603fd5b
+  metadata.gz: 83b73a10113e75106a0fb7af9bec79802d2e3f5c8f3e07742f33a52642a9441c20769072f8ea5bd532011b7d172db6ca007121d6874f705008e1a5a511ca1ff8
+  data.tar.gz: d9c53588e8adbd76764c20012baf702591276d84c2cd64ed0bb0d5b742699607a2287d30a2fb4f70c1bf6f8a7338d716d989b208c8983c2a87eeddbd6d96dd3d

data/.travis.yml

@@ -7,6 +7,7 @@ rvm:
   - 2.4.4
   - 2.5.1
   - 2.6.1
+  - 2.7.0
 # avoid having travis install jdk on MRI builds where we don't need it.
 matrix:
   include:

data/CHANGES.md

@@ -6,7 +6,9 @@
 
 *
 
-*
+## 3.4.0
+
+* XML-mode `extract_xpath` now supports extracting attribute values with xpath @attr syntax.
 
 ## 3.3.0
 

data/doc/xml.md

@@ -72,6 +72,16 @@ You can use all the standard transforation macros in Traject::Macros::Transforma
 to_field "something", extract_xpath("//value"), first_only, translation_map("some_map"), default("no value")
 ```
 
+### selecting attribute values
+
+Just works, using xpath syntax for selecting an attribute:
+
+
+```ruby
+# gets status value in:  <oai:header status="something">
+to_field "status", extract_xpath("//oai:record/oai:header/@status")
+```
+
 
 ### selecting non-text nodes
 

data/lib/traject/macros/marc21.rb

@@ -42,11 +42,11 @@ module Traject::Macros
     #
     # * :translation_map => String: translate with named translation map looked up in load
     #       path, uses Tranject::TranslationMap.new(translation_map_arg).
-    #       **Instead**, use `extract_marc(whatever), translation_map(translation_map_arg)
+    #       **Instead**, use `extract_marc(whatever), translation_map(translation_map_arg)`
     #
     # * :trim_punctuation => true; trims leading/trailing punctuation using standard algorithms that
     #     have shown themselves useful with Marc, using Marc21.trim_punctuation. **Instead**, use
-    #    `extract_marc(whatever), trim_punctuation
+    #    `extract_marc(whatever), trim_punctuation`
     #
     # * :default => String: if otherwise empty, add default value. **Instead**, use `extract_marc(whatever), default("default value")`
     #

data/lib/traject/macros/nokogiri_macros.rb

@@ -26,9 +26,15 @@ module Traject
             # Make sure to avoid text content that was all blank, which is "between the children"
             # whitespace.
             result = result.collect do |n|
-              n.xpath('.//text()').collect(&:text).tap do |arr|
-                arr.reject! { |s| s =~ (/\A\s+\z/) }
-              end.join(" ")
+              if n.kind_of?(Nokogiri::XML::Attr)
+                # attribute value
+                n.value
+              else
+                # text from node
+                n.xpath('.//text()').collect(&:text).tap do |arr|
+                  arr.reject! { |s| s =~ (/\A\s+\z/) }
+                end.join(" ")
+              end
             end
           else
             # just put all matches in accumulator as Nokogiri::XML::Node's

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.3.0"
+  VERSION = "3.4.0"
 end

data/lib/translation_maps/marc_languages.yaml

@@ -10,18 +10,21 @@ ady: Adygei
 afa: Afroasiatic (Other)
 afh: Afrihili (Artificial language)
 afr: Afrikaans
-ajm: Aljamia
+ain: Ainu
+ajm: Aljamía
 aka: Akan
 akk: Akkadian
 alb: Albanian
 ale: Aleut
 alg: Algonquian (Other)
+alt: Altai
 amh: Amharic
-ang: English, Old (ca. 450-1100)
+ang: "English, Old (ca. 450-1100)"
+anp: Angika
 apa: Apache languages
 ara: Arabic
 arc: Aramaic
-arg: Aragonese Spanish
+arg: Aragonese
 arm: Armenian
 arn: Mapuche
 arp: Arapaho
@@ -36,7 +39,7 @@ ave: Avestan
 awa: Awadhi
 aym: Aymara
 aze: Azerbaijani
-bad: Banda
+bad: Banda languages
 bai: Bamileke languages
 bak: Bashkir
 bal: Baluchi
@@ -51,7 +54,7 @@ bem: Bemba
 ben: Bengali
 ber: Berber (Other)
 bho: Bhojpuri
-bih: Bihari
+bih: Bihari (Other)
 bik: Bikol
 bin: Edo
 bis: Bislama
@@ -65,6 +68,7 @@ bua: Buriat
 bug: Bugis
 bul: Bulgarian
 bur: Burmese
+byn: Bilin
 cad: Caddo
 cai: Central American Indian (Other)
 cam: Khmer
@@ -78,7 +82,7 @@ chb: Chibcha
 che: Chechen
 chg: Chagatai
 chi: Chinese
-chk: Truk
+chk: Chuukese
 chm: Mari
 chn: Chinook jargon
 cho: Choctaw
@@ -88,15 +92,17 @@ chu: Church Slavic
 chv: Chuvash
 chy: Cheyenne
 cmc: Chamic languages
+cnr: Montenegrin
 cop: Coptic
 cor: Cornish
 cos: Corsican
-cpe: Creoles and Pidgins, English-based (Other)
-cpf: Creoles and Pidgins, French-based (Other)
-cpp: Creoles and Pidgins, Portuguese-based (Other)
+cpe: "Creoles and Pidgins, English-based (Other)"
+cpf: "Creoles and Pidgins, French-based (Other)"
+cpp: "Creoles and Pidgins, Portuguese-based (Other)"
 cre: Cree
 crh: Crimean Tatar
 crp: Creoles and Pidgins (Other)
+csb: Kashubian
 cus: Cushitic (Other)
 cze: Czech
 dak: Dakota
@@ -104,14 +110,15 @@ dan: Danish
 dar: Dargwa
 day: Dayak
 del: Delaware
-den: Slave
+den: Slavey
 dgr: Dogrib
 din: Dinka
 div: Divehi
 doi: Dogri
 dra: Dravidian (Other)
+dsb: Lower Sorbian
 dua: Duala
-dum: Dutch, Middle (ca. 1050-1350)
+dum: "Dutch, Middle (ca. 1050-1350)"
 dut: Dutch
 dyu: Dyula
 dzo: Dzongkha
@@ -120,7 +127,7 @@ egy: Egyptian
 eka: Ekajuk
 elx: Elamite
 eng: English
-enm: English, Middle (1100-1500)
+enm: "English, Middle (1100-1500)"
 epo: Esperanto
 esk: Eskimo languages
 esp: Esperanto
@@ -133,18 +140,21 @@ fao: Faroese
 far: Faroese
 fat: Fanti
 fij: Fijian
+fil: Filipino
 fin: Finnish
 fiu: Finno-Ugrian (Other)
 fon: Fon
 fre: French
 fri: Frisian
-frm: French, Middle (ca. 1400-1600)
-fro: French, Old (ca. 842-1400)
+frm: "French, Middle (ca. 1300-1600)"
+fro: "French, Old (ca. 842-1300)"
+frr: North Frisian
+frs: East Frisian
 fry: Frisian
 ful: Fula
 fur: Friulian
-gaa: Ga
-gae: Scottish Gaelic
+gaa: Gã
+gae: Scottish Gaelix
 gag: Galician
 gal: Oromo
 gay: Gayo
@@ -158,15 +168,16 @@ gla: Scottish Gaelic
 gle: Irish
 glg: Galician
 glv: Manx
-gmh: German, Middle High (ca. 1050-1500)
-goh: German, Old High (ca. 750-1050)
+gmh: "German, Middle High (ca. 1050-1500)"
+goh: "German, Old High (ca. 750-1050)"
 gon: Gondi
 gor: Gorontalo
 got: Gothic
 grb: Grebo
-grc: Greek, Ancient (to 1453)
-gre: Greek, Modern (1453- )
+grc: "Greek, Ancient (to 1453)"
+gre: "Greek, Modern (1453-)"
 grn: Guarani
+gsw: Swiss German
 gua: Guarani
 guj: Gujarati
 gwi: Gwich'in
@@ -177,11 +188,13 @@ haw: Hawaiian
 heb: Hebrew
 her: Herero
 hil: Hiligaynon
-him: Himachali
+him: Western Pahari languages
 hin: Hindi
 hit: Hittite
 hmn: Hmong
 hmo: Hiri Motu
+hrv: Croatian
+hsb: Upper Sorbian
 hun: Hungarian
 hup: Hupa
 iba: Iban
@@ -205,16 +218,17 @@ iri: Irish
 iro: Iroquoian (Other)
 ita: Italian
 jav: Javanese
+jbo: Lojban (Artificial language)
 jpn: Japanese
 jpr: Judeo-Persian
 jrb: Judeo-Arabic
 kaa: Kara-Kalpak
 kab: Kabyle
 kac: Kachin
-kal: Kalatdlisut
+kal: Kalâtdlisut
 kam: Kamba
 kan: Kannada
-kar: Karen
+kar: Karen languages
 kas: Kashmiri
 kau: Kanuri
 kaw: Kawi
@@ -232,19 +246,21 @@ kok: Konkani
 kom: Komi
 kon: Kongo
 kor: Korean
-kos: Kusaie
+kos: Kosraean
 kpe: Kpelle
-kro: Kru
+krc: Karachay-Balkar
+krl: Karelian
+kro: Kru (Other)
 kru: Kurukh
 kua: Kuanyama
 kum: Kumyk
 kur: Kurdish
 kus: Kusaie
-kut: Kutenai
+kut: Kootenai
 lad: Ladino
-lah: Lahnda
-lam: Lamba
-lan: Occitan (post-1500)
+lah: Lahndā
+lam: Lamba (Zambia and Congo)
+lan: Occitan (post 1500)
 lao: Lao
 lap: Sami
 lat: Latin
@@ -255,11 +271,11 @@ lin: Lingala
 lit: Lithuanian
 lol: Mongo-Nkundu
 loz: Lozi
-ltz: Letzeburgesch
+ltz: Luxembourgish
 lua: Luba-Lulua
 lub: Luba-Katanga
 lug: Ganda
-lui: Luiseno
+lui: Luiseño
 lun: Lunda
 luo: Luo (Kenya and Tanzania)
 lus: Lushai
@@ -274,12 +290,13 @@ man: Mandingo
 mao: Maori
 map: Austronesian (Other)
 mar: Marathi
-mas: Masai
+mas: Maasai
 max: Manx
 may: Malay
+mdf: Moksha
 mdr: Mandar
 men: Mende
-mga: Irish, Middle (ca. 1100-1550)
+mga: "Irish, Middle (ca. 1100-1550)"
 mic: Micmac
 min: Minangkabau
 mis: Miscellaneous languages
@@ -293,12 +310,14 @@ mno: Manobo languages
 moh: Mohawk
 mol: Moldavian
 mon: Mongolian
-mos: Moore
+mos: Mooré
 mul: Multiple languages
 mun: Munda (Other)
 mus: Creek
+mwl: Mirandese
 mwr: Marwari
 myn: Mayan languages
+myv: Erzya
 nah: Nahuatl
 nai: North American Indian (Other)
 nap: Neapolitan Italian
@@ -314,12 +333,14 @@ nia: Nias
 nic: Niger-Kordofanian (Other)
 niu: Niuean
 nno: Norwegian (Nynorsk)
-nob: Norwegian (Bokmal)
+nob: Norwegian (Bokmål)
 nog: Nogai
 non: Old Norse
 nor: Norwegian
+nqo: N'Ko
 nso: Northern Sotho
 nub: Nubian languages
+nwc: "Newari, Old"
 nya: Nyanja
 nym: Nyamwezi
 nyn: Nyankole
@@ -331,7 +352,7 @@ ori: Oriya
 orm: Oromo
 osa: Osage
 oss: Ossetic
-ota: Turkish, Ottoman
+ota: "Turkish, Ottoman"
 oto: Otomian languages
 paa: Papuan (Other)
 pag: Pangasinan
@@ -346,10 +367,10 @@ phi: Philippine (Other)
 phn: Phoenician
 pli: Pali
 pol: Polish
-pon: Ponape
+pon: Pohnpeian
 por: Portuguese
 pra: Prakrit languages
-pro: Provencal (to 1500)
+pro: Provençal (to 1500)
 pus: Pushto
 que: Quechua
 raj: Rajasthani
@@ -360,6 +381,7 @@ roh: Raeto-Romance
 rom: Romani
 rum: Romanian
 run: Rundi
+rup: Aromanian
 rus: Russian
 sad: Sandawe
 sag: Sango (Ubangi Creole)
@@ -372,11 +394,12 @@ sao: Samoan
 sas: Sasak
 sat: Santali
 scc: Serbian
+scn: Sicilian Italian
 sco: Scots
 scr: Croatian
 sel: Selkup
 sem: Semitic (Other)
-sga: Irish, Old (to 1100)
+sga: "Irish, Old (to 1100)"
 sgn: Sign languages
 shn: Shan
 sho: Shona
@@ -404,6 +427,8 @@ son: Songhai
 sot: Sotho
 spa: Spanish
 srd: Sardinian
+srn: Sranan
+srp: Serbian
 srr: Serer
 ssa: Nilo-Saharan (Other)
 sso: Sotho
@@ -415,7 +440,8 @@ sux: Sumerian
 swa: Swahili
 swe: Swedish
 swz: Swazi
-syr: Syriac
+syc: Syriac
+syr: "Syriac, Modern"
 tag: Tagalog
 tah: Tahitian
 tai: Tai (Other)
@@ -431,10 +457,11 @@ tgk: Tajik
 tgl: Tagalog
 tha: Thai
 tib: Tibetan
-tig: Tigre
+tig: Tigré
 tir: Tigrinya
 tiv: Tiv
 tkl: Tokelauan
+tlh: Klingon (Artificial language)
 tli: Tlingit
 tmh: Tamashek
 tog: Tonga (Nyasa)
@@ -464,17 +491,17 @@ uzb: Uzbek
 vai: Vai
 ven: Venda
 vie: Vietnamese
-vol: Volapuk
+vol: Volapük
 vot: Votic
 wak: Wakashan languages
-wal: Walamo
+wal: Wolayta
 war: Waray
-was: Washo
+was: Washoe
 wel: Welsh
-wen: Sorbian languages
+wen: Sorbian (Other)
 wln: Walloon
 wol: Wolof
-xal: Kalmyk
+xal: Oirat
 xho: Xhosa
 yao: Yao (Africa)
 yap: Yapese
@@ -482,9 +509,11 @@ yid: Yiddish
 yor: Yoruba
 ypk: Yupik languages
 zap: Zapotec
+zbl: Blissymbolics
 zen: Zenaga
 zha: Zhuang
-znd: Zande
+znd: Zande languages
 zul: Zulu
 zun: Zuni
-# zxx: null
+# zxx: No linguistic content
+zza: Zaza

data/test/indexer/nokogiri_indexer_test.rb

@@ -109,6 +109,41 @@ describe "Traject::NokogiriIndexer" do
         result["name"].name == "name"
       })
     end
+  end
 
+  describe "xpath to attribute" do
+    let(:indexer) do
+      namespaces = @namespaces
+      Traject::Indexer::NokogiriIndexer.new("nokogiri.namespaces" => namespaces,
+                                            "nokogiri.each_record_xpath" => "//oai:record") do
+        to_field "status", extract_xpath("//oai:record/oai:header/@status")
+      end
+    end
+
+    let(:records) { Traject::NokogiriReader.new(StringIO.new(
+                                                      <<-XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+        <responseDate>2020-03-03T04:16:09Z</responseDate>
+        <request verb="ListRecords" metadataPrefix="marc21" set="blacklight" from="2020-03-02T20:47:11Z">https://na02.alma.exlibrisgroup.com/view/oai/01TULI_INST/request</request>
+        <ListRecords>
+          <record>
+            <header status="deleted">
+              <identifier>oai:alma.01TULI_INST:991025803889703811</identifier>
+              <datestamp>2020-03-03T03:54:35Z</datestamp>
+              <setSpec>blacklight</setSpec>
+              <setSpec>rapid_print_journals</setSpec>
+              <setSpec>blacklight_qa</setSpec>
+            </header>
+          </record>
+        </ListRecords>
+      </OAI-PMH>
+          XML
+    ), []).to_a }
+
+    it "extracts the correct attribute" do
+      statuses = indexer.map_record(records.first)["status"]
+      assert_equal ["deleted"], statuses
+    end
   end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 - Bill Dueber
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-02 00:00:00.000000000 Z
+date: 2020-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -231,7 +231,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
-description: 
+description:
 email:
 - none@nowhere.org
 executables:
@@ -390,7 +390,7 @@ homepage: http://github.com/traject/traject
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -405,9 +405,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
   system, focused on library-archives-museums input, and indexing to Solr as output.