traject 0.0.2 → 0.9.1

data/lib/translation_maps/marc_languages.yaml

@@ -0,0 +1,490 @@
+# Map Language Codes (in 008[35-37], 041) to User Friendly Term
+
+# ???: null
+aar: Afar
+abk: Abkhaz
+ace: Achinese
+ach: Acoli
+ada: Adangme
+ady: Adygei
+afa: Afroasiatic (Other)
+afh: Afrihili (Artificial language)
+afr: Afrikaans
+ajm: Aljamia
+aka: Akan
+akk: Akkadian
+alb: Albanian
+ale: Aleut
+alg: Algonquian (Other)
+amh: Amharic
+ang: English, Old (ca. 450-1100)
+apa: Apache languages
+ara: Arabic
+arc: Aramaic
+arg: Aragonese Spanish
+arm: Armenian
+arn: Mapuche
+arp: Arapaho
+art: Artificial (Other)
+arw: Arawak
+asm: Assamese
+ast: Bable
+ath: Athapascan (Other)
+aus: Australian languages
+ava: Avaric
+ave: Avestan
+awa: Awadhi
+aym: Aymara
+aze: Azerbaijani
+bad: Banda
+bai: Bamileke languages
+bak: Bashkir
+bal: Baluchi
+bam: Bambara
+ban: Balinese
+baq: Basque
+bas: Basa
+bat: Baltic (Other)
+bej: Beja
+bel: Belarusian
+bem: Bemba
+ben: Bengali
+ber: Berber (Other)
+bho: Bhojpuri
+bih: Bihari
+bik: Bikol
+bin: Edo
+bis: Bislama
+bla: Siksika
+bnt: Bantu (Other)
+bos: Bosnian
+bra: Braj
+bre: Breton
+btk: Batak
+bua: Buriat
+bug: Bugis
+bul: Bulgarian
+bur: Burmese
+cad: Caddo
+cai: Central American Indian (Other)
+cam: Khmer
+car: Carib
+cat: Catalan
+cau: Caucasian (Other)
+ceb: Cebuano
+cel: Celtic (Other)
+cha: Chamorro
+chb: Chibcha
+che: Chechen
+chg: Chagatai
+chi: Chinese
+chk: Truk
+chm: Mari
+chn: Chinook jargon
+cho: Choctaw
+chp: Chipewyan
+chr: Cherokee
+chu: Church Slavic
+chv: Chuvash
+chy: Cheyenne
+cmc: Chamic languages
+cop: Coptic
+cor: Cornish
+cos: Corsican
+cpe: Creoles and Pidgins, English-based (Other)
+cpf: Creoles and Pidgins, French-based (Other)
+cpp: Creoles and Pidgins, Portuguese-based (Other)
+cre: Cree
+crh: Crimean Tatar
+crp: Creoles and Pidgins (Other)
+cus: Cushitic (Other)
+cze: Czech
+dak: Dakota
+dan: Danish
+dar: Dargwa
+day: Dayak
+del: Delaware
+den: Slave
+dgr: Dogrib
+din: Dinka
+div: Divehi
+doi: Dogri
+dra: Dravidian (Other)
+dua: Duala
+dum: Dutch, Middle (ca. 1050-1350)
+dut: Dutch
+dyu: Dyula
+dzo: Dzongkha
+efi: Efik
+egy: Egyptian
+eka: Ekajuk
+elx: Elamite
+eng: English
+enm: English, Middle (1100-1500)
+epo: Esperanto
+esk: Eskimo languages
+esp: Esperanto
+est: Estonian
+eth: Ethiopic
+ewe: Ewe
+ewo: Ewondo
+fan: Fang
+fao: Faroese
+far: Faroese
+fat: Fanti
+fij: Fijian
+fin: Finnish
+fiu: Finno-Ugrian (Other)
+fon: Fon
+fre: French
+fri: Frisian
+frm: French, Middle (ca. 1400-1600)
+fro: French, Old (ca. 842-1400)
+fry: Frisian
+ful: Fula
+fur: Friulian
+gaa: Ga
+gae: Scottish Gaelic
+gag: Galician
+gal: Oromo
+gay: Gayo
+gba: Gbaya
+gem: Germanic (Other)
+geo: Georgian
+ger: German
+gez: Ethiopic
+gil: Gilbertese
+gla: Scottish Gaelic
+gle: Irish
+glg: Galician
+glv: Manx
+gmh: German, Middle High (ca. 1050-1500)
+goh: German, Old High (ca. 750-1050)
+gon: Gondi
+gor: Gorontalo
+got: Gothic
+grb: Grebo
+grc: Greek, Ancient (to 1453)
+gre: Greek, Modern (1453- )
+grn: Guarani
+gua: Guarani
+guj: Gujarati
+gwi: Gwich'in
+hai: Haida
+hat: Haitian French Creole
+hau: Hausa
+haw: Hawaiian
+heb: Hebrew
+her: Herero
+hil: Hiligaynon
+him: Himachali
+hin: Hindi
+hit: Hittite
+hmn: Hmong
+hmo: Hiri Motu
+hun: Hungarian
+hup: Hupa
+iba: Iban
+ibo: Igbo
+ice: Icelandic
+ido: Ido
+iii: Sichuan Yi
+ijo: Ijo
+iku: Inuktitut
+ile: Interlingue
+ilo: Iloko
+ina: Interlingua (International Auxiliary Language Association)
+inc: Indic (Other)
+ind: Indonesian
+ine: Indo-European (Other)
+inh: Ingush
+int: Interlingua (International Auxiliary Language Association)
+ipk: Inupiaq
+ira: Iranian (Other)
+iri: Irish
+iro: Iroquoian (Other)
+ita: Italian
+jav: Javanese
+jpn: Japanese
+jpr: Judeo-Persian
+jrb: Judeo-Arabic
+kaa: Kara-Kalpak
+kab: Kabyle
+kac: Kachin
+kal: Kalatdlisut
+kam: Kamba
+kan: Kannada
+kar: Karen
+kas: Kashmiri
+kau: Kanuri
+kaw: Kawi
+kaz: Kazakh
+kbd: Kabardian
+kha: Khasi
+khi: Khoisan (Other)
+khm: Khmer
+kho: Khotanese
+kik: Kikuyu
+kin: Kinyarwanda
+kir: Kyrgyz
+kmb: Kimbundu
+kok: Konkani
+kom: Komi
+kon: Kongo
+kor: Korean
+kos: Kusaie
+kpe: Kpelle
+kro: Kru
+kru: Kurukh
+kua: Kuanyama
+kum: Kumyk
+kur: Kurdish
+kus: Kusaie
+kut: Kutenai
+lad: Ladino
+lah: Lahnda
+lam: Lamba
+lan: Occitan (post-1500)
+lao: Lao
+lap: Sami
+lat: Latin
+lav: Latvian
+lez: Lezgian
+lim: Limburgish
+lin: Lingala
+lit: Lithuanian
+lol: Mongo-Nkundu
+loz: Lozi
+ltz: Letzeburgesch
+lua: Luba-Lulua
+lub: Luba-Katanga
+lug: Ganda
+lui: Luiseno
+lun: Lunda
+luo: Luo (Kenya and Tanzania)
+lus: Lushai
+mac: Macedonian
+mad: Madurese
+mag: Magahi
+mah: Marshallese
+mai: Maithili
+mak: Makasar
+mal: Malayalam
+man: Mandingo
+mao: Maori
+map: Austronesian (Other)
+mar: Marathi
+mas: Masai
+max: Manx
+may: Malay
+mdr: Mandar
+men: Mende
+mga: Irish, Middle (ca. 1100-1550)
+mic: Micmac
+min: Minangkabau
+mis: Miscellaneous languages
+mkh: Mon-Khmer (Other)
+mla: Malagasy
+mlg: Malagasy
+mlt: Maltese
+mnc: Manchu
+mni: Manipuri
+mno: Manobo languages
+moh: Mohawk
+mol: Moldavian
+mon: Mongolian
+mos: Moore
+mul: Multiple languages
+mun: Munda (Other)
+mus: Creek
+mwr: Marwari
+myn: Mayan languages
+nah: Nahuatl
+nai: North American Indian (Other)
+nap: Neapolitan Italian
+nau: Nauru
+nav: Navajo
+nbl: Ndebele (South Africa)
+nde: Ndebele (Zimbabwe)
+ndo: Ndonga
+nds: Low German
+nep: Nepali
+new: Newari
+nia: Nias
+nic: Niger-Kordofanian (Other)
+niu: Niuean
+nno: Norwegian (Nynorsk)
+nob: Norwegian (Bokmal)
+nog: Nogai
+non: Old Norse
+nor: Norwegian
+nso: Northern Sotho
+nub: Nubian languages
+nya: Nyanja
+nym: Nyamwezi
+nyn: Nyankole
+nyo: Nyoro
+nzi: Nzima
+oci: Occitan (post-1500)
+oji: Ojibwa
+ori: Oriya
+orm: Oromo
+osa: Osage
+oss: Ossetic
+ota: Turkish, Ottoman
+oto: Otomian languages
+paa: Papuan (Other)
+pag: Pangasinan
+pal: Pahlavi
+pam: Pampanga
+pan: Panjabi
+pap: Papiamento
+pau: Palauan
+peo: Old Persian (ca. 600-400 B.C.)
+per: Persian
+phi: Philippine (Other)
+phn: Phoenician
+pli: Pali
+pol: Polish
+pon: Ponape
+por: Portuguese
+pra: Prakrit languages
+pro: Provencal (to 1500)
+pus: Pushto
+que: Quechua
+raj: Rajasthani
+rap: Rapanui
+rar: Rarotongan
+roa: Romance (Other)
+roh: Raeto-Romance
+rom: Romani
+rum: Romanian
+run: Rundi
+rus: Russian
+sad: Sandawe
+sag: Sango (Ubangi Creole)
+sah: Yakut
+sai: South American Indian (Other)
+sal: Salishan languages
+sam: Samaritan Aramaic
+san: Sanskrit
+sao: Samoan
+sas: Sasak
+sat: Santali
+scc: Serbian
+sco: Scots
+scr: Croatian
+sel: Selkup
+sem: Semitic (Other)
+sga: Irish, Old (to 1100)
+sgn: Sign languages
+shn: Shan
+sho: Shona
+sid: Sidamo
+sin: Sinhalese
+sio: Siouan (Other)
+sit: Sino-Tibetan (Other)
+sla: Slavic (Other)
+slo: Slovak
+slv: Slovenian
+sma: Southern Sami
+sme: Northern Sami
+smi: Sami
+smj: Lule Sami
+smn: Inari Sami
+smo: Samoan
+sms: Skolt Sami
+sna: Shona
+snd: Sindhi
+snh: Sinhalese
+snk: Soninke
+sog: Sogdian
+som: Somali
+son: Songhai
+sot: Sotho
+spa: Spanish
+srd: Sardinian
+srr: Serer
+ssa: Nilo-Saharan (Other)
+sso: Sotho
+ssw: Swazi
+suk: Sukuma
+sun: Sundanese
+sus: Susu
+sux: Sumerian
+swa: Swahili
+swe: Swedish
+swz: Swazi
+syr: Syriac
+tag: Tagalog
+tah: Tahitian
+tai: Tai (Other)
+taj: Tajik
+tam: Tamil
+tar: Tatar
+tat: Tatar
+tel: Telugu
+tem: Temne
+ter: Terena
+tet: Tetum
+tgk: Tajik
+tgl: Tagalog
+tha: Thai
+tib: Tibetan
+tig: Tigre
+tir: Tigrinya
+tiv: Tiv
+tkl: Tokelauan
+tli: Tlingit
+tmh: Tamashek
+tog: Tonga (Nyasa)
+ton: Tongan
+tpi: Tok Pisin
+tru: Truk
+tsi: Tsimshian
+tsn: Tswana
+tso: Tsonga
+tsw: Tswana
+tuk: Turkmen
+tum: Tumbuka
+tup: Tupi languages
+tur: Turkish
+tut: Altaic (Other)
+tvl: Tuvaluan
+twi: Twi
+tyv: Tuvinian
+udm: Udmurt
+uga: Ugaritic
+uig: Uighur
+ukr: Ukrainian
+umb: Umbundu
+# und: Undetermined
+urd: Urdu
+uzb: Uzbek
+vai: Vai
+ven: Venda
+vie: Vietnamese
+vol: Volapuk
+vot: Votic
+wak: Wakashan languages
+wal: Walamo
+war: Waray
+was: Washo
+wel: Welsh
+wen: Sorbian languages
+wln: Walloon
+wol: Wolof
+xal: Kalmyk
+xho: Xhosa
+yao: Yao (Africa)
+yap: Yapese
+yid: Yiddish
+yor: Yoruba
+ypk: Yupik languages
+zap: Zapotec
+zen: Zenaga
+zha: Zhuang
+znd: Zande
+zul: Zulu
+zun: Zuni
+# zxx: null

data/test/indexer/each_record_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+describe "Traject::Indexer#each_record" do
+  before do
+    @indexer = Traject::Indexer.new
+  end
+
+  describe "checks arguments" do
+    it "rejects no-arg block" do
+      assert_raises(ArgumentError) do
+        @indexer.each_record do
+        end
+      end
+    end
+    it "rejects three-arg block" do
+      assert_raises(ArgumentError) do
+        @indexer.each_record do |one, two, three|
+        end
+      end
+    end
+    it "accepts one-arg block" do
+      @indexer.each_record do |record|
+      end
+    end
+    it "accepts two-arg block" do
+      @indexer.each_record do |record, context|
+      end
+    end
+    it "accepts variable arity block" do
+      @indexer.each_record do |*variable|
+      end
+    end
+  end
+end

data/test/indexer/macros_marc21_semantics_test.rb

@@ -0,0 +1,206 @@
+require 'test_helper'
+
+require 'traject/indexer'
+require 'traject/macros/marc21_semantics'
+
+require 'json'
+require 'marc/record'
+
+# See also marc_extractor_test.rb for more detailed tests on marc extraction,
+# this is just a basic test to make sure our macro works passing through to there
+# and other options.
+describe "Traject::Macros::Marc21Semantics" do
+  Marc21Semantics = Traject::Macros::Marc21Semantics # shortcut
+
+  before do
+    @indexer = Traject::Indexer.new
+    @indexer.extend Marc21Semantics
+
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+  end
+
+  it "oclcnum" do
+    @indexer.instance_eval do
+      to_field "oclcnum", oclcnum
+    end
+    output = @indexer.map_record(@record)
+
+    assert_equal %w{2710183 47971712},  output["oclcnum"]
+  end
+
+  describe "marc_sortable_author" do
+    # these probably should be taking only certain subfields, but we're copying
+    # from SolrMarc that didn't do so either and nobody noticed, so not bothering for now.
+    before do
+      @indexer.instance_eval do
+        to_field "author_sort", marc_sortable_author
+      end
+    end
+    it "collates author and title" do
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Herman, Edward S.Manufacturing consent : the political economy of the mass media / Edward S. Herman and Noam Chomsky ; with a new introduction by the authors."], output["author_sort"]
+    end
+    it "respects non-filing" do
+      @record = MARC::Reader.new(support_file_path  "the_business_ren.marc").to_a.first
+
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Business renaissance quarterly [electronic resource]."], output["author_sort"]
+    end
+  end
+
+  describe "marc_sortable_title" do
+    before do
+      @indexer.instance_eval { to_field "title_sort", marc_sortable_title }
+    end
+    it "works" do
+      output = @indexer.map_record(@record)
+      assert_equal ["Manufacturing consent : the political economy of the mass media"], output["title_sort"]
+    end
+    it "respects non-filing" do
+      @record = MARC::Reader.new(support_file_path  "the_business_ren.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Business renaissance quarterly"], output["title_sort"]
+    end
+    it "works with a record with no 245$ab" do
+      @record = MARC::Reader.new(support_file_path  "245_no_ab.marc").to_a.first
+      output = @indexer.map_record(@record)
+      assert_equal ["Papers"], output["title_sort"]
+    end
+  end
+
+  describe "marc_languages" do
+    before do
+      @indexer.instance_eval {to_field "languages", marc_languages() }
+    end
+
+    it "unpacks packed 041a and translates" do
+      @record = MARC::Reader.new(support_file_path  "packed_041a_lang.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["English", "French", "German", "Italian", "Spanish", "Russian"], output["languages"]
+    end
+  end
+
+  describe "marc_instrumentation_humanized" do
+    before do
+      @record = MARC::Reader.new(support_file_path  "musical_cage.marc").to_a.first
+      @indexer.instance_eval {to_field "instrumentation", marc_instrumentation_humanized }
+    end
+
+    it "translates, de-duping" do
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Larger ensemble, Unspecified", "Piano", "Soprano voice", "Tenor voice", "Violin", "Larger ensemble, Ethnic", "Guitar", "Voices, Unspecified"], output["instrumentation"]
+    end
+  end
+
+  describe "marc_instrument_codes_normalized" do
+    before do
+      @record = MARC::Reader.new(support_file_path  "musical_cage.marc").to_a.first
+      @indexer.instance_eval {to_field "instrument_codes", marc_instrument_codes_normalized }
+    end
+    it "normalizes, de-duping" do
+      output = @indexer.map_record(@record)
+
+      assert_equal ["on", "ka01", "ka", "va01", "va", "vd01", "vd", "sa01", "sa", "oy", "tb01", "tb", "vn12", "vn"],
+        output["instrument_codes"]
+    end
+    it "codes soloist 048$b" do
+      @record = MARC::Reader.new(support_file_path  "louis_armstrong.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["bb01", "bb01.s", "bb", "bb.s", "oe"],
+        output["instrument_codes"]
+    end
+  end
+
+  describe "publication_date" do
+    # there are way too many edge cases for us to test em all, but we'll test some of em.
+    it "pulls out 008 date_type s" do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      assert_equal 2002, Marc21Semantics.publication_date(@record)
+    end
+    it "uses start date for date_type c continuing resource" do
+      @record = MARC::Reader.new(support_file_path  "the_business_ren.marc").to_a.first
+      assert_equal 2006, Marc21Semantics.publication_date(@record)
+    end
+    it "returns nil when the records really got nothing" do
+      @record = MARC::Reader.new(support_file_path  "emptyish_record.marc").to_a.first
+      assert_equal nil, Marc21Semantics.publication_date(@record)
+    end
+    it "estimates with a single 'u'" do
+      @record = MARC::Reader.new(support_file_path  "date_with_u.marc").to_a.first
+      # was 184u as date1 on a continuing resource. For continuing resources,
+      # we take the first date. And need to deal with the u.
+      assert_equal 1845, Marc21Semantics.publication_date(@record)
+    end
+    it "resorts to 260c" do
+      @record = MARC::Reader.new(support_file_path  "date_resort_to_260.marc").to_a.first
+      assert_equal 1980, Marc21Semantics.publication_date(@record)
+    end
+    it "works with date type r missing date2" do
+      @record = MARC::Reader.new(support_file_path  "date_type_r_missing_date2.marc").to_a.first
+      assert_equal 1957, Marc21Semantics.publication_date(@record)
+    end
+  end
+
+  describe "marc_lcc_to_broad_category" do
+    before do
+      @indexer.instance_eval {to_field "discipline_facet", marc_lcc_to_broad_category }
+    end
+    it "maps a simple example" do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Language & Literature"], output["discipline_facet"]
+    end
+    it "maps to default" do
+      @record = MARC::Reader.new(support_file_path  "musical_cage.marc").to_a.first
+      output = @indexer.map_record(@record)
+      assert_equal ["Unknown"], output["discipline_facet"]
+    end
+    it "maps to nothing if none and no default" do
+      @indexer.instance_eval {to_field "discipline_no_default", marc_lcc_to_broad_category(:default => nil)}
+      @record = MARC::Reader.new(support_file_path  "musical_cage.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_nil output["discipline_no_default"]
+    end
+  end
+
+  describe "marc_geo_facet" do
+    before do
+      @indexer.instance_eval {to_field "geo_facet", marc_geo_facet }
+    end
+    it "maps a complicated record" do
+      @record = MARC::Reader.new(support_file_path  "multi_geo.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Europe", "Middle East", "Africa, North", "Agora (Athens, Greece)", "Rome (Italy)", "Italy"], 
+        output["geo_facet"]
+    end
+    it "maps nothing on a record with no geo" do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      output = @indexer.map_record(@record)
+      assert_nil output["geo_facet"]
+    end
+  end
+
+  describe "marc_era_facet" do
+    before do
+      @indexer.instance_eval {to_field "era_facet", marc_era_facet}
+    end
+    it "maps a complicated record" do
+      @record = MARC::Reader.new(support_file_path  "multi_era.marc").to_a.first
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Early modern, 1500-1700", "17th century", "Great Britain: Puritan Revolution, 1642-1660", "Great Britain: Civil War, 1642-1649", "1642-1660"],
+        output["era_facet"]
+    end
+
+  end
+
+end