traject 3.3.0 → 3.7.0

data/lib/translation_maps/marc_languages.yaml

@@ -10,18 +10,21 @@ ady: Adygei
 afa: Afroasiatic (Other)
 afh: Afrihili (Artificial language)
 afr: Afrikaans
-ajm: Aljamia
+ain: Ainu
+ajm: Aljamía
 aka: Akan
 akk: Akkadian
 alb: Albanian
 ale: Aleut
 alg: Algonquian (Other)
+alt: Altai
 amh: Amharic
-ang: English, Old (ca. 450-1100)
+ang: "English, Old (ca. 450-1100)"
+anp: Angika
 apa: Apache languages
 ara: Arabic
 arc: Aramaic
-arg: Aragonese Spanish
+arg: Aragonese
 arm: Armenian
 arn: Mapuche
 arp: Arapaho
@@ -36,7 +39,7 @@ ave: Avestan
 awa: Awadhi
 aym: Aymara
 aze: Azerbaijani
-bad: Banda
+bad: Banda languages
 bai: Bamileke languages
 bak: Bashkir
 bal: Baluchi
@@ -51,7 +54,7 @@ bem: Bemba
 ben: Bengali
 ber: Berber (Other)
 bho: Bhojpuri
-bih: Bihari
+bih: Bihari (Other)
 bik: Bikol
 bin: Edo
 bis: Bislama
@@ -65,6 +68,7 @@ bua: Buriat
 bug: Bugis
 bul: Bulgarian
 bur: Burmese
+byn: Bilin
 cad: Caddo
 cai: Central American Indian (Other)
 cam: Khmer
@@ -78,7 +82,7 @@ chb: Chibcha
 che: Chechen
 chg: Chagatai
 chi: Chinese
-chk: Truk
+chk: Chuukese
 chm: Mari
 chn: Chinook jargon
 cho: Choctaw
@@ -88,15 +92,17 @@ chu: Church Slavic
 chv: Chuvash
 chy: Cheyenne
 cmc: Chamic languages
+cnr: Montenegrin
 cop: Coptic
 cor: Cornish
 cos: Corsican
-cpe: Creoles and Pidgins, English-based (Other)
-cpf: Creoles and Pidgins, French-based (Other)
-cpp: Creoles and Pidgins, Portuguese-based (Other)
+cpe: "Creoles and Pidgins, English-based (Other)"
+cpf: "Creoles and Pidgins, French-based (Other)"
+cpp: "Creoles and Pidgins, Portuguese-based (Other)"
 cre: Cree
 crh: Crimean Tatar
 crp: Creoles and Pidgins (Other)
+csb: Kashubian
 cus: Cushitic (Other)
 cze: Czech
 dak: Dakota
@@ -104,14 +110,15 @@ dan: Danish
 dar: Dargwa
 day: Dayak
 del: Delaware
-den: Slave
+den: Slavey
 dgr: Dogrib
 din: Dinka
 div: Divehi
 doi: Dogri
 dra: Dravidian (Other)
+dsb: Lower Sorbian
 dua: Duala
-dum: Dutch, Middle (ca. 1050-1350)
+dum: "Dutch, Middle (ca. 1050-1350)"
 dut: Dutch
 dyu: Dyula
 dzo: Dzongkha
@@ -120,7 +127,7 @@ egy: Egyptian
 eka: Ekajuk
 elx: Elamite
 eng: English
-enm: English, Middle (1100-1500)
+enm: "English, Middle (1100-1500)"
 epo: Esperanto
 esk: Eskimo languages
 esp: Esperanto
@@ -133,18 +140,21 @@ fao: Faroese
 far: Faroese
 fat: Fanti
 fij: Fijian
+fil: Filipino
 fin: Finnish
 fiu: Finno-Ugrian (Other)
 fon: Fon
 fre: French
 fri: Frisian
-frm: French, Middle (ca. 1400-1600)
-fro: French, Old (ca. 842-1400)
+frm: "French, Middle (ca. 1300-1600)"
+fro: "French, Old (ca. 842-1300)"
+frr: North Frisian
+frs: East Frisian
 fry: Frisian
 ful: Fula
 fur: Friulian
-gaa: Ga
-gae: Scottish Gaelic
+gaa: Gã
+gae: Scottish Gaelix
 gag: Galician
 gal: Oromo
 gay: Gayo
@@ -158,15 +168,16 @@ gla: Scottish Gaelic
 gle: Irish
 glg: Galician
 glv: Manx
-gmh: German, Middle High (ca. 1050-1500)
-goh: German, Old High (ca. 750-1050)
+gmh: "German, Middle High (ca. 1050-1500)"
+goh: "German, Old High (ca. 750-1050)"
 gon: Gondi
 gor: Gorontalo
 got: Gothic
 grb: Grebo
-grc: Greek, Ancient (to 1453)
-gre: Greek, Modern (1453- )
+grc: "Greek, Ancient (to 1453)"
+gre: "Greek, Modern (1453-)"
 grn: Guarani
+gsw: Swiss German
 gua: Guarani
 guj: Gujarati
 gwi: Gwich'in
@@ -177,11 +188,13 @@ haw: Hawaiian
 heb: Hebrew
 her: Herero
 hil: Hiligaynon
-him: Himachali
+him: Western Pahari languages
 hin: Hindi
 hit: Hittite
 hmn: Hmong
 hmo: Hiri Motu
+hrv: Croatian
+hsb: Upper Sorbian
 hun: Hungarian
 hup: Hupa
 iba: Iban
@@ -205,16 +218,17 @@ iri: Irish
 iro: Iroquoian (Other)
 ita: Italian
 jav: Javanese
+jbo: Lojban (Artificial language)
 jpn: Japanese
 jpr: Judeo-Persian
 jrb: Judeo-Arabic
 kaa: Kara-Kalpak
 kab: Kabyle
 kac: Kachin
-kal: Kalatdlisut
+kal: Kalâtdlisut
 kam: Kamba
 kan: Kannada
-kar: Karen
+kar: Karen languages
 kas: Kashmiri
 kau: Kanuri
 kaw: Kawi
@@ -232,19 +246,21 @@ kok: Konkani
 kom: Komi
 kon: Kongo
 kor: Korean
-kos: Kusaie
+kos: Kosraean
 kpe: Kpelle
-kro: Kru
+krc: Karachay-Balkar
+krl: Karelian
+kro: Kru (Other)
 kru: Kurukh
 kua: Kuanyama
 kum: Kumyk
 kur: Kurdish
 kus: Kusaie
-kut: Kutenai
+kut: Kootenai
 lad: Ladino
-lah: Lahnda
-lam: Lamba
-lan: Occitan (post-1500)
+lah: Lahndā
+lam: Lamba (Zambia and Congo)
+lan: Occitan (post 1500)
 lao: Lao
 lap: Sami
 lat: Latin
@@ -255,11 +271,11 @@ lin: Lingala
 lit: Lithuanian
 lol: Mongo-Nkundu
 loz: Lozi
-ltz: Letzeburgesch
+ltz: Luxembourgish
 lua: Luba-Lulua
 lub: Luba-Katanga
 lug: Ganda
-lui: Luiseno
+lui: Luiseño
 lun: Lunda
 luo: Luo (Kenya and Tanzania)
 lus: Lushai
@@ -274,12 +290,13 @@ man: Mandingo
 mao: Maori
 map: Austronesian (Other)
 mar: Marathi
-mas: Masai
+mas: Maasai
 max: Manx
 may: Malay
+mdf: Moksha
 mdr: Mandar
 men: Mende
-mga: Irish, Middle (ca. 1100-1550)
+mga: "Irish, Middle (ca. 1100-1550)"
 mic: Micmac
 min: Minangkabau
 mis: Miscellaneous languages
@@ -293,12 +310,14 @@ mno: Manobo languages
 moh: Mohawk
 mol: Moldavian
 mon: Mongolian
-mos: Moore
+mos: Mooré
 mul: Multiple languages
 mun: Munda (Other)
 mus: Creek
+mwl: Mirandese
 mwr: Marwari
 myn: Mayan languages
+myv: Erzya
 nah: Nahuatl
 nai: North American Indian (Other)
 nap: Neapolitan Italian
@@ -314,12 +333,14 @@ nia: Nias
 nic: Niger-Kordofanian (Other)
 niu: Niuean
 nno: Norwegian (Nynorsk)
-nob: Norwegian (Bokmal)
+nob: Norwegian (Bokmål)
 nog: Nogai
 non: Old Norse
 nor: Norwegian
+nqo: N'Ko
 nso: Northern Sotho
 nub: Nubian languages
+nwc: "Newari, Old"
 nya: Nyanja
 nym: Nyamwezi
 nyn: Nyankole
@@ -331,7 +352,7 @@ ori: Oriya
 orm: Oromo
 osa: Osage
 oss: Ossetic
-ota: Turkish, Ottoman
+ota: "Turkish, Ottoman"
 oto: Otomian languages
 paa: Papuan (Other)
 pag: Pangasinan
@@ -346,10 +367,10 @@ phi: Philippine (Other)
 phn: Phoenician
 pli: Pali
 pol: Polish
-pon: Ponape
+pon: Pohnpeian
 por: Portuguese
 pra: Prakrit languages
-pro: Provencal (to 1500)
+pro: Provençal (to 1500)
 pus: Pushto
 que: Quechua
 raj: Rajasthani
@@ -360,6 +381,7 @@ roh: Raeto-Romance
 rom: Romani
 rum: Romanian
 run: Rundi
+rup: Aromanian
 rus: Russian
 sad: Sandawe
 sag: Sango (Ubangi Creole)
@@ -372,11 +394,12 @@ sao: Samoan
 sas: Sasak
 sat: Santali
 scc: Serbian
+scn: Sicilian Italian
 sco: Scots
 scr: Croatian
 sel: Selkup
 sem: Semitic (Other)
-sga: Irish, Old (to 1100)
+sga: "Irish, Old (to 1100)"
 sgn: Sign languages
 shn: Shan
 sho: Shona
@@ -404,6 +427,8 @@ son: Songhai
 sot: Sotho
 spa: Spanish
 srd: Sardinian
+srn: Sranan
+srp: Serbian
 srr: Serer
 ssa: Nilo-Saharan (Other)
 sso: Sotho
@@ -415,7 +440,8 @@ sux: Sumerian
 swa: Swahili
 swe: Swedish
 swz: Swazi
-syr: Syriac
+syc: Syriac
+syr: "Syriac, Modern"
 tag: Tagalog
 tah: Tahitian
 tai: Tai (Other)
@@ -431,10 +457,11 @@ tgk: Tajik
 tgl: Tagalog
 tha: Thai
 tib: Tibetan
-tig: Tigre
+tig: Tigré
 tir: Tigrinya
 tiv: Tiv
 tkl: Tokelauan
+tlh: Klingon (Artificial language)
 tli: Tlingit
 tmh: Tamashek
 tog: Tonga (Nyasa)
@@ -464,17 +491,17 @@ uzb: Uzbek
 vai: Vai
 ven: Venda
 vie: Vietnamese
-vol: Volapuk
+vol: Volapük
 vot: Votic
 wak: Wakashan languages
-wal: Walamo
+wal: Wolayta
 war: Waray
-was: Washo
+was: Washoe
 wel: Welsh
-wen: Sorbian languages
+wen: Sorbian (Other)
 wln: Walloon
 wol: Wolof
-xal: Kalmyk
+xal: Oirat
 xho: Xhosa
 yao: Yao (Africa)
 yap: Yapese
@@ -482,9 +509,11 @@ yid: Yiddish
 yor: Yoruba
 ypk: Yupik languages
 zap: Zapotec
+zbl: Blissymbolics
 zen: Zenaga
 zha: Zhuang
-znd: Zande
+znd: Zande languages
 zul: Zulu
 zun: Zuni
-# zxx: null
+# zxx: No linguistic content
+zza: Zaza

data/test/command_line_test.rb

@@ -0,0 +1,52 @@
+# we mostly unit test with a Traject::Indexer itself and lower-level, but
+# we need at least some basic top-level integration actually command line tests,
+# this is a start, we can add more.
+#
+# Should we be testing Traject::CommandLine as an object instead of/in addition to
+# actually testing shell-out to command line call? Maybe.
+
+require 'test_helper'
+
+describe "Shell out to command line" do
+  # just encapsuluate using the minitest capture helper, but also
+  # getting and returning exit code
+  #
+  #     out, err, result = execute_with_args("-c configuration")
+  def execute_with_args(args)
+    out, err = capture_subprocess_io do
+      system("./bin/traject #{args}")
+    end
+
+    return out, err, $?
+  end
+
+  it "can display version" do
+    out, err, result = execute_with_args("-v")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert_equal err, "traject version #{Traject::VERSION}\n"
+  end
+
+  it "can display help text" do
+    out, err, result = execute_with_args("-h")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert err.start_with?("traject [options] -c configuration.rb [-c config2.rb] file.mrc")
+  end
+
+  it "handles bad argument" do
+    out, err, result = execute_with_args("--no-such-arg")
+    refute result.success?
+
+    assert err.start_with?("Error: unknown option `--no-such-arg'\nExiting...\n")
+  end
+
+  it "does basic dry run" do
+    out, err, result = execute_with_args("--debug-mode -s one=two -s three=four -c test/test_support/demo_config.rb test/test_support/emptyish_record.marc")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert_includes err, "executing with: `--debug-mode -s one=two -s three=four"
+    assert_match /bib_1000165 +author_sort +Collection la/, out
+  end
+end
+

data/test/debug_writer_test.rb

@@ -73,6 +73,19 @@ describe 'Simple output' do
 
   end
 
+  it "deals ok with nil values" do
+    record_with_nil_value = {"id"=>["2710183"], "title"=>["Manufacturing consent : the political economy of the mass media /"], "xyz"=>nil}
+    @writer.put Traject::Indexer::Context.new(:output_hash => record_with_nil_value)
+    expected = [
+      "#{@id} id #{@id}",
+      "#{@id} title #{@title}",
+      "#{@id} xyz",
+      "\n"
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    @writer.close
+
+  end
 end
 
 

data/test/indexer/macros/macros_marc21_semantics_test.rb

@@ -209,6 +209,10 @@ describe "Traject::Macros::Marc21Semantics" do
       @record = MARC::Reader.new(support_file_path  "date_type_r_missing_date2.marc").to_a.first
       assert_equal 1957, Marc21Semantics.publication_date(@record)
     end
+    it "provides a fallback for a missing second date" do
+      @record = MARC::Reader.new(support_file_path  "missing-second-date.marc").to_a.first
+      assert_equal 1678, Marc21Semantics.publication_date(@record)
+    end
 
     it "works correctly with date type 'q'" do
       val = @record['008'].value

data/test/indexer/macros/transformation_test.rb

@@ -174,4 +174,114 @@ describe "Traject::Macros::Transformation" do
     end
   end
 
+  describe "delete_if" do
+
+    describe "argument is an Array" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "filters out selected values from accumulatd values" do
+        arg = [ "one", "three"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "filters out selected values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "filters out selected values from accumulatd values" do
+        arg = ->(v) { v == "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), delete_if(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
+  describe "select" do
+
+    describe "argument is an Array" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"]
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Set" do
+      it "selects a subset of values from accumulatd values" do
+        arg = [ "one", "three", "four"].to_set
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["one", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Regex" do
+      it "selects a subset of values from accumulatd values" do
+        arg = /^t/
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+
+    describe "argument is a Procedure or Lambda" do
+      it "selects a subset of values from accumulatd values" do
+        arg = ->(v) { v != "one" }
+
+        @indexer.configure do
+          to_field "test", literal("one"), literal("two"), literal("three"), select(arg)
+        end
+
+        output = @indexer.map_record(@record)
+        assert_equal ["two", "three"], output["test"]
+      end
+    end
+  end
+
 end

data/test/indexer/nokogiri_indexer_test.rb

@@ -109,6 +109,41 @@ describe "Traject::NokogiriIndexer" do
         result["name"].name == "name"
       })
     end
+  end
 
+  describe "xpath to attribute" do
+    let(:indexer) do
+      namespaces = @namespaces
+      Traject::Indexer::NokogiriIndexer.new("nokogiri.namespaces" => namespaces,
+                                            "nokogiri.each_record_xpath" => "//oai:record") do
+        to_field "status", extract_xpath("//oai:record/oai:header/@status")
+      end
+    end
+
+    let(:records) { Traject::NokogiriReader.new(StringIO.new(
+                                                      <<-XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+        <responseDate>2020-03-03T04:16:09Z</responseDate>
+        <request verb="ListRecords" metadataPrefix="marc21" set="blacklight" from="2020-03-02T20:47:11Z">https://na02.alma.exlibrisgroup.com/view/oai/01TULI_INST/request</request>
+        <ListRecords>
+          <record>
+            <header status="deleted">
+              <identifier>oai:alma.01TULI_INST:991025803889703811</identifier>
+              <datestamp>2020-03-03T03:54:35Z</datestamp>
+              <setSpec>blacklight</setSpec>
+              <setSpec>rapid_print_journals</setSpec>
+              <setSpec>blacklight_qa</setSpec>
+            </header>
+          </record>
+        </ListRecords>
+      </OAI-PMH>
+          XML
+    ), []).to_a }
+
+    it "extracts the correct attribute" do
+      statuses = indexer.map_record(records.first)["status"]
+      assert_equal ["deleted"], statuses
+    end
   end
 end

data/test/indexer/read_write_test.rb

@@ -7,7 +7,8 @@ memory_writer_class = Class.new do
       # store them in a class variable so we can test em later
       # Supress the warning message
       original_verbose, $VERBOSE = $VERBOSE, nil
-      @@last_writer_settings = @settings = settings
+      @settings = settings
+      self.class.store_last_writer_settings(@settings)
       # Activate warning messages again.
       $VERBOSE = original_verbose
       @settings["memory_writer.added"] = []
@@ -20,6 +21,16 @@ memory_writer_class = Class.new do
     def close
       @settings["memory_writer.closed"] = true
     end
+
+    private
+
+    def self.store_last_writer_settings(settings)
+      @last_writer_settings = settings
+    end
+
+    def self.last_writer_settings
+      @last_writer_settings
+    end
   end
 
 describe "Traject::Indexer#process" do
@@ -53,7 +64,7 @@ describe "Traject::Indexer#process" do
 
     # Grab the settings out of a class variable where we left em,
     # as a convenient place to store outcomes so we can test em.
-    writer_settings = memory_writer_class.class_variable_get("@@last_writer_settings")
+    writer_settings = memory_writer_class.last_writer_settings
 
     assert writer_settings["memory_writer.added"]
     assert_equal 30, writer_settings["memory_writer.added"].length
@@ -146,7 +157,7 @@ describe "Traject::Indexer#process" do
     it "parses and loads" do
       @indexer.process([@file1, @file2])
       # kinda ridic, yeah.
-      output_hashes = memory_writer_class.class_variable_get("@@last_writer_settings")["memory_writer.added"].collect(&:output_hash)
+      output_hashes = memory_writer_class.last_writer_settings["memory_writer.added"].collect(&:output_hash)
 
       assert_length 2, output_hashes
       assert output_hashes.all? { |hash| hash["title"].length > 0 }