traject 3.1.0.rc1 → 3.5.0

data/lib/traject/oai_pmh_nokogiri_reader.rb

@@ -115,9 +115,15 @@ module Traject
     # @returns [HTTP::Client] from http.rb gem
     def http_client
       @http_client ||= begin
-        # timeout setting on http.rb seems to be a mess.
-        # https://github.com/httprb/http/issues/488
-        client = HTTP.timeout(:global, write: timeout / 3, connect: timeout / 3, read: timeout / 3)
+        client = nil
+
+        if HTTP::VERSION.split(".").first.to_i > 3
+          client = HTTP.timeout(timeout)
+        else
+          # timeout setting on http.rb 3.x are a bit of a mess.
+          # https://github.com/httprb/http/issues/488
+          client = HTTP.timeout(:global, write: timeout / 3, connect: timeout / 3, read: timeout / 3)
+        end
 
         if settings["oai_pmh.try_gzip"]
           client = client.use(:auto_inflate).headers("accept-encoding" => "gzip;q=1.0, identity;q=0.5")

data/lib/traject/solr_json_writer.rb

@@ -41,10 +41,12 @@ require 'concurrent' # for atomic_fixnum
 #
 # ## Relevant settings
 #
-# * solr.url (optional if solr.update_url is set) The URL to the solr core to index into
+# * solr.url (optional if solr.update_url is set) The URL to the solr core to index into.
+#   (Can include embedded HTTP basic auth as eg `http://user:pass@host/solr`)
 #
 # * solr.update_url: The actual update url. If unset, we'll first see if
-#   "#{solr.url}/update/json" exists, and if not use "#{solr.url}/update"
+#   "#{solr.url}/update/json" exists, and if not use "#{solr.url}/update". (Can include
+#   embedded HTTP basic auth as eg `http://user:pass@host/solr)
 #
 # * solr_writer.batch_size: How big a batch to send to solr. Default is 100.
 #   My tests indicate that this setting doesn't change overall index speed by a ton.
@@ -101,12 +103,17 @@ class Traject::SolrJsonWriter
   def initialize(argSettings)
     @settings = Traject::Indexer::Settings.new(argSettings)
 
+
     # Set max errors
     @max_skipped = (@settings['solr_writer.max_skipped'] || DEFAULT_MAX_SKIPPED).to_i
     if @max_skipped < 0
       @max_skipped = nil
     end
 
+
+    # Figure out where to send updates, and if with basic auth
+    @solr_update_url, basic_auth_user, basic_auth_password = self.determine_solr_update_url
+
     @http_client = if @settings["solr_json_writer.http_client"]
       @settings["solr_json_writer.http_client"]
     else
@@ -114,6 +121,11 @@ class Traject::SolrJsonWriter
       if @settings["solr_writer.http_timeout"]
         client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
       end
+
+      if basic_auth_user || basic_auth_password
+        client.set_auth(@solr_update_url, basic_auth_user, basic_auth_password)
+      end
+
       client
     end
 
@@ -137,13 +149,11 @@ class Traject::SolrJsonWriter
     # this the new default writer.
     @commit_on_close = (settings["solr_writer.commit_on_close"] || settings["solrj_writer.commit_on_close"]).to_s == "true"
 
-    # Figure out where to send updates
-    @solr_update_url = self.determine_solr_update_url
 
     @solr_update_args = settings["solr_writer.solr_update_args"]
     @commit_solr_update_args = settings["solr_writer.commit_solr_update_args"]
 
-    logger.info("   #{self.class.name} writing to '#{@solr_update_url}' in batches of #{@batch_size} with #{@thread_pool_size} bg threads")
+    logger.info("   #{self.class.name} writing to '#{@solr_update_url}' #{"(with HTTP basic auth)" if basic_auth_user || basic_auth_password}in batches of #{@batch_size} with #{@thread_pool_size} bg threads")
   end
 
 
@@ -185,6 +195,9 @@ class Traject::SolrJsonWriter
   # @param [Array<Traject::Indexer::Context>] an array of contexts
   def send_batch(batch)
     return if batch.empty?
+
+    logger.debug("#{self.class.name}: sending batch of #{batch.size} to Solr")
+
     json_package = JSON.generate(batch.map { |c| c.output_hash })
 
     begin
@@ -209,12 +222,15 @@ class Traject::SolrJsonWriter
   # Send a single context to Solr, logging an error if need be
   # @param [Traject::Indexer::Context] c The context whose document you want to send
   def send_single(c)
+    logger.debug("#{self.class.name}: sending single record to Solr: #{c.output_hash}")
+
     json_package = JSON.generate([c.output_hash])
     begin
-      resp = @http_client.post solr_update_url_with_query(@solr_update_args), json_package, "Content-type" => "application/json"
+      post_url = solr_update_url_with_query(@solr_update_args)
+      resp = @http_client.post post_url, json_package, "Content-type" => "application/json"
 
       unless resp.status == 200
-        raise BadHttpResponse.new("Unexpected HTTP response status #{resp.status}", resp)
+        raise BadHttpResponse.new("Unexpected HTTP response status #{resp.status} from POST #{post_url}", resp)
       end
 
       # Catch Timeouts and network errors -- as well as non-200 http responses --
@@ -234,7 +250,7 @@ class Traject::SolrJsonWriter
       if @max_skipped and skipped_record_count > @max_skipped
         # re-raising in rescue means the last encountered error will be available as #cause
         # on raised exception, a feature in ruby 2.1+.
-        raise MaxSkippedRecordsExceeded.new("#{self.class.name}: Exceeded maximum number of skipped records (#{@max_skipped}): aborting")
+        raise MaxSkippedRecordsExceeded.new("#{self.class.name}: Exceeded maximum number of skipped records (#{@max_skipped}): aborting: #{exception.message}")
       end
     end
   end
@@ -255,6 +271,8 @@ class Traject::SolrJsonWriter
   # There is no built-in way to direct a record to be deleted from an indexing config
   # file at the moment, this is just a loose method on the writer.
   def delete(id)
+    logger.debug("#{self.class.name}: Sending delete to Solr for #{id}")
+
     json_package = {delete: id}
     resp = @http_client.post solr_update_url_with_query(@solr_update_args), JSON.generate(json_package), "Content-type" => "application/json"
     if resp.status != 200
@@ -262,6 +280,13 @@ class Traject::SolrJsonWriter
     end
   end
 
+  # Send a delete all query.
+  #
+  # This method takes no params and will not automatically commit the deletes.
+  # @example @writer.delete_all!
+  def delete_all!
+    delete(query: "*:*")
+  end
 
   # Get the logger from the settings, or default to an effectively null logger
   def logger
@@ -282,14 +307,16 @@ class Traject::SolrJsonWriter
       @thread_pool.maybe_in_thread_pool { send_batch(batch) }
     end
 
-    # Wait for shutdown, and time it.
-    logger.debug "#{self.class.name}: Shutting down thread pool, waiting if needed..."
-    elapsed = @thread_pool.shutdown_and_wait
-    if elapsed > 60
-      logger.warn "Waited #{elapsed} seconds for all threads, you may want to increase solr_writer.thread_pool (currently #{@settings["solr_writer.thread_pool"]})"
+    if @thread_pool_size && @thread_pool_size > 0
+      # Wait for shutdown, and time it.
+      logger.debug "#{self.class.name}: Shutting down thread pool, waiting if needed..."
+      elapsed = @thread_pool.shutdown_and_wait
+      if elapsed > 60
+        logger.warn "Waited #{elapsed} seconds for all threads, you may want to increase solr_writer.thread_pool (currently #{@settings["solr_writer.thread_pool"]})"
+      end
+      logger.debug "#{self.class.name}: Thread pool shutdown complete"
+      logger.warn "#{self.class.name}: #{skipped_record_count} skipped records" if skipped_record_count > 0
     end
-    logger.debug "#{self.class.name}: Thread pool shutdown complete"
-    logger.warn "#{self.class.name}: #{skipped_record_count} skipped records" if skipped_record_count > 0
 
     # check again now that we've waited, there could still be some
     # that didn't show up before.
@@ -345,13 +372,27 @@ class Traject::SolrJsonWriter
   end
 
 
-  # Relatively complex logic to determine if we have a valid URL and what it is
+  # Relatively complex logic to determine if we have a valid URL and what it is,
+  # and if we have basic_auth info
+  #
+  # Empties out user and password embedded in URI returned, to help avoid logging it.
+  #
+  # @returns [update_url, basic_auth_user, basic_auth_password]
   def determine_solr_update_url
-    if settings['solr.update_url']
+    url = if settings['solr.update_url']
       check_solr_update_url(settings['solr.update_url'])
     else
       derive_solr_update_url_from_solr_url(settings['solr.url'])
     end
+
+    parsed_uri                            = URI.parse(url)
+    user_from_uri, password_from_uri      = parsed_uri.user, parsed_uri.password
+    parsed_uri.user, parsed_uri.password  = nil, nil
+
+    basic_auth_user     = @settings["solr_writer.basic_auth_user"] || user_from_uri
+    basic_auth_password = @settings["solr_writer.basic_auth_password"] || password_from_uri
+
+    return [parsed_uri.to_s, basic_auth_user, basic_auth_password]
   end
 
 

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "3.1.0.rc1"
+  VERSION = "3.5.0"
 end

data/lib/translation_maps/marc_languages.yaml

@@ -10,18 +10,21 @@ ady: Adygei
 afa: Afroasiatic (Other)
 afh: Afrihili (Artificial language)
 afr: Afrikaans
-ajm: Aljamia
+ain: Ainu
+ajm: Aljamía
 aka: Akan
 akk: Akkadian
 alb: Albanian
 ale: Aleut
 alg: Algonquian (Other)
+alt: Altai
 amh: Amharic
-ang: English, Old (ca. 450-1100)
+ang: "English, Old (ca. 450-1100)"
+anp: Angika
 apa: Apache languages
 ara: Arabic
 arc: Aramaic
-arg: Aragonese Spanish
+arg: Aragonese
 arm: Armenian
 arn: Mapuche
 arp: Arapaho
@@ -36,7 +39,7 @@ ave: Avestan
 awa: Awadhi
 aym: Aymara
 aze: Azerbaijani
-bad: Banda
+bad: Banda languages
 bai: Bamileke languages
 bak: Bashkir
 bal: Baluchi
@@ -51,7 +54,7 @@ bem: Bemba
 ben: Bengali
 ber: Berber (Other)
 bho: Bhojpuri
-bih: Bihari
+bih: Bihari (Other)
 bik: Bikol
 bin: Edo
 bis: Bislama
@@ -65,6 +68,7 @@ bua: Buriat
 bug: Bugis
 bul: Bulgarian
 bur: Burmese
+byn: Bilin
 cad: Caddo
 cai: Central American Indian (Other)
 cam: Khmer
@@ -78,7 +82,7 @@ chb: Chibcha
 che: Chechen
 chg: Chagatai
 chi: Chinese
-chk: Truk
+chk: Chuukese
 chm: Mari
 chn: Chinook jargon
 cho: Choctaw
@@ -88,15 +92,17 @@ chu: Church Slavic
 chv: Chuvash
 chy: Cheyenne
 cmc: Chamic languages
+cnr: Montenegrin
 cop: Coptic
 cor: Cornish
 cos: Corsican
-cpe: Creoles and Pidgins, English-based (Other)
-cpf: Creoles and Pidgins, French-based (Other)
-cpp: Creoles and Pidgins, Portuguese-based (Other)
+cpe: "Creoles and Pidgins, English-based (Other)"
+cpf: "Creoles and Pidgins, French-based (Other)"
+cpp: "Creoles and Pidgins, Portuguese-based (Other)"
 cre: Cree
 crh: Crimean Tatar
 crp: Creoles and Pidgins (Other)
+csb: Kashubian
 cus: Cushitic (Other)
 cze: Czech
 dak: Dakota
@@ -104,14 +110,15 @@ dan: Danish
 dar: Dargwa
 day: Dayak
 del: Delaware
-den: Slave
+den: Slavey
 dgr: Dogrib
 din: Dinka
 div: Divehi
 doi: Dogri
 dra: Dravidian (Other)
+dsb: Lower Sorbian
 dua: Duala
-dum: Dutch, Middle (ca. 1050-1350)
+dum: "Dutch, Middle (ca. 1050-1350)"
 dut: Dutch
 dyu: Dyula
 dzo: Dzongkha
@@ -120,7 +127,7 @@ egy: Egyptian
 eka: Ekajuk
 elx: Elamite
 eng: English
-enm: English, Middle (1100-1500)
+enm: "English, Middle (1100-1500)"
 epo: Esperanto
 esk: Eskimo languages
 esp: Esperanto
@@ -133,18 +140,21 @@ fao: Faroese
 far: Faroese
 fat: Fanti
 fij: Fijian
+fil: Filipino
 fin: Finnish
 fiu: Finno-Ugrian (Other)
 fon: Fon
 fre: French
 fri: Frisian
-frm: French, Middle (ca. 1400-1600)
-fro: French, Old (ca. 842-1400)
+frm: "French, Middle (ca. 1300-1600)"
+fro: "French, Old (ca. 842-1300)"
+frr: North Frisian
+frs: East Frisian
 fry: Frisian
 ful: Fula
 fur: Friulian
-gaa: Ga
-gae: Scottish Gaelic
+gaa: Gã
+gae: Scottish Gaelix
 gag: Galician
 gal: Oromo
 gay: Gayo
@@ -158,15 +168,16 @@ gla: Scottish Gaelic
 gle: Irish
 glg: Galician
 glv: Manx
-gmh: German, Middle High (ca. 1050-1500)
-goh: German, Old High (ca. 750-1050)
+gmh: "German, Middle High (ca. 1050-1500)"
+goh: "German, Old High (ca. 750-1050)"
 gon: Gondi
 gor: Gorontalo
 got: Gothic
 grb: Grebo
-grc: Greek, Ancient (to 1453)
-gre: Greek, Modern (1453- )
+grc: "Greek, Ancient (to 1453)"
+gre: "Greek, Modern (1453-)"
 grn: Guarani
+gsw: Swiss German
 gua: Guarani
 guj: Gujarati
 gwi: Gwich'in
@@ -177,11 +188,13 @@ haw: Hawaiian
 heb: Hebrew
 her: Herero
 hil: Hiligaynon
-him: Himachali
+him: Western Pahari languages
 hin: Hindi
 hit: Hittite
 hmn: Hmong
 hmo: Hiri Motu
+hrv: Croatian
+hsb: Upper Sorbian
 hun: Hungarian
 hup: Hupa
 iba: Iban
@@ -205,16 +218,17 @@ iri: Irish
 iro: Iroquoian (Other)
 ita: Italian
 jav: Javanese
+jbo: Lojban (Artificial language)
 jpn: Japanese
 jpr: Judeo-Persian
 jrb: Judeo-Arabic
 kaa: Kara-Kalpak
 kab: Kabyle
 kac: Kachin
-kal: Kalatdlisut
+kal: Kalâtdlisut
 kam: Kamba
 kan: Kannada
-kar: Karen
+kar: Karen languages
 kas: Kashmiri
 kau: Kanuri
 kaw: Kawi
@@ -232,19 +246,21 @@ kok: Konkani
 kom: Komi
 kon: Kongo
 kor: Korean
-kos: Kusaie
+kos: Kosraean
 kpe: Kpelle
-kro: Kru
+krc: Karachay-Balkar
+krl: Karelian
+kro: Kru (Other)
 kru: Kurukh
 kua: Kuanyama
 kum: Kumyk
 kur: Kurdish
 kus: Kusaie
-kut: Kutenai
+kut: Kootenai
 lad: Ladino
-lah: Lahnda
-lam: Lamba
-lan: Occitan (post-1500)
+lah: Lahndā
+lam: Lamba (Zambia and Congo)
+lan: Occitan (post 1500)
 lao: Lao
 lap: Sami
 lat: Latin
@@ -255,11 +271,11 @@ lin: Lingala
 lit: Lithuanian
 lol: Mongo-Nkundu
 loz: Lozi
-ltz: Letzeburgesch
+ltz: Luxembourgish
 lua: Luba-Lulua
 lub: Luba-Katanga
 lug: Ganda
-lui: Luiseno
+lui: Luiseño
 lun: Lunda
 luo: Luo (Kenya and Tanzania)
 lus: Lushai
@@ -274,12 +290,13 @@ man: Mandingo
 mao: Maori
 map: Austronesian (Other)
 mar: Marathi
-mas: Masai
+mas: Maasai
 max: Manx
 may: Malay
+mdf: Moksha
 mdr: Mandar
 men: Mende
-mga: Irish, Middle (ca. 1100-1550)
+mga: "Irish, Middle (ca. 1100-1550)"
 mic: Micmac
 min: Minangkabau
 mis: Miscellaneous languages
@@ -293,12 +310,14 @@ mno: Manobo languages
 moh: Mohawk
 mol: Moldavian
 mon: Mongolian
-mos: Moore
+mos: Mooré
 mul: Multiple languages
 mun: Munda (Other)
 mus: Creek
+mwl: Mirandese
 mwr: Marwari
 myn: Mayan languages
+myv: Erzya
 nah: Nahuatl
 nai: North American Indian (Other)
 nap: Neapolitan Italian
@@ -314,12 +333,14 @@ nia: Nias
 nic: Niger-Kordofanian (Other)
 niu: Niuean
 nno: Norwegian (Nynorsk)
-nob: Norwegian (Bokmal)
+nob: Norwegian (Bokmål)
 nog: Nogai
 non: Old Norse
 nor: Norwegian
+nqo: N'Ko
 nso: Northern Sotho
 nub: Nubian languages
+nwc: "Newari, Old"
 nya: Nyanja
 nym: Nyamwezi
 nyn: Nyankole
@@ -331,7 +352,7 @@ ori: Oriya
 orm: Oromo
 osa: Osage
 oss: Ossetic
-ota: Turkish, Ottoman
+ota: "Turkish, Ottoman"
 oto: Otomian languages
 paa: Papuan (Other)
 pag: Pangasinan
@@ -346,10 +367,10 @@ phi: Philippine (Other)
 phn: Phoenician
 pli: Pali
 pol: Polish
-pon: Ponape
+pon: Pohnpeian
 por: Portuguese
 pra: Prakrit languages
-pro: Provencal (to 1500)
+pro: Provençal (to 1500)
 pus: Pushto
 que: Quechua
 raj: Rajasthani
@@ -360,6 +381,7 @@ roh: Raeto-Romance
 rom: Romani
 rum: Romanian
 run: Rundi
+rup: Aromanian
 rus: Russian
 sad: Sandawe
 sag: Sango (Ubangi Creole)
@@ -372,11 +394,12 @@ sao: Samoan
 sas: Sasak
 sat: Santali
 scc: Serbian
+scn: Sicilian Italian
 sco: Scots
 scr: Croatian
 sel: Selkup
 sem: Semitic (Other)
-sga: Irish, Old (to 1100)
+sga: "Irish, Old (to 1100)"
 sgn: Sign languages
 shn: Shan
 sho: Shona
@@ -404,6 +427,8 @@ son: Songhai
 sot: Sotho
 spa: Spanish
 srd: Sardinian
+srn: Sranan
+srp: Serbian
 srr: Serer
 ssa: Nilo-Saharan (Other)
 sso: Sotho
@@ -415,7 +440,8 @@ sux: Sumerian
 swa: Swahili
 swe: Swedish
 swz: Swazi
-syr: Syriac
+syc: Syriac
+syr: "Syriac, Modern"
 tag: Tagalog
 tah: Tahitian
 tai: Tai (Other)
@@ -431,10 +457,11 @@ tgk: Tajik
 tgl: Tagalog
 tha: Thai
 tib: Tibetan
-tig: Tigre
+tig: Tigré
 tir: Tigrinya
 tiv: Tiv
 tkl: Tokelauan
+tlh: Klingon (Artificial language)
 tli: Tlingit
 tmh: Tamashek
 tog: Tonga (Nyasa)
@@ -464,17 +491,17 @@ uzb: Uzbek
 vai: Vai
 ven: Venda
 vie: Vietnamese
-vol: Volapuk
+vol: Volapük
 vot: Votic
 wak: Wakashan languages
-wal: Walamo
+wal: Wolayta
 war: Waray
-was: Washo
+was: Washoe
 wel: Welsh
-wen: Sorbian languages
+wen: Sorbian (Other)
 wln: Walloon
 wol: Wolof
-xal: Kalmyk
+xal: Oirat
 xho: Xhosa
 yao: Yao (Africa)
 yap: Yapese
@@ -482,9 +509,11 @@ yid: Yiddish
 yor: Yoruba
 ypk: Yupik languages
 zap: Zapotec
+zbl: Blissymbolics
 zen: Zenaga
 zha: Zhuang
-znd: Zande
+znd: Zande languages
 zul: Zulu
 zun: Zuni
-# zxx: null
+# zxx: No linguistic content
+zza: Zaza