tinycus 1.0.0

data/tinycus.rb

@@ -0,0 +1,1514 @@
+# coding: utf-8
+require 'digest'
+require 'json'
+
+module Tinycus
+
+  # The four "alpha_" functions work on Greek and English, also most Latin characters; see comments in Tr.get_greek_collation_tr.
+
+  def Tinycus.alpha_sort(l,n:false)
+    return Tinycus.sort(l,Tinycus.alpha_collation,n:false)
+  end
+
+  def Tinycus.alpha_equal(a,b,n:true)
+    return (Tinycus.alpha_compare(a,b,n:n)==0)
+  end
+
+  def Tinycus.alpha_compare(a,b,n:true)
+    # return (Tinycus::Tr.remove_accents_from_greek(a,n:n).downcase <=> Tinycus::Tr.remove_accents_from_greek(b,n:n).downcase)
+    collation_tr = Tinycus.alpha_collation
+    return collation_tr.apply(a,n:n) <=> collation_tr.apply(b,n:n)
+  end
+
+  def Tinycus.alpha_collation
+    return Tinycus::Tr.get_greek_collation_tr
+  end
+
+  def Tinycus.is_vowel(c)
+    # synonym of Tinycus.contains_vowel, for readability when using it on a single character
+    return Tinycus.contains_vowel(c)
+  end
+
+  def Tinycus.contains_vowel(s)
+    # works for Greek and Latin; considers y to be a vowel; doesn't handle stuff like Welsh w
+    if Tinycus::Tr.remove_accents_from_greek(s).downcase.match?(/[αειουηωaeiouyæ]/) then return true else return false end
+  end
+
+  # ---
+
+  def Tinycus.sort(l,collation_tr,n:false)
+    p = l.clone # This does work on a list of strings: ruby -e "a=['p','q']; b=a.clone; b[1]='x'; print a"
+    p.sort_by! { |x| collation_tr.apply(x,n:n) } # ruby's sort_by! only applies the block once to each element, to form an index
+    return p
+  end
+
+  def Tinycus.run_tests
+    # removing accents:
+    tests = [
+      ['',''],
+      ['ἔχω','εχω'],
+    ]
+    tests.each { |t|
+      x,y = t
+      z = Tinycus::Tr.remove_accents_from_greek(x)
+      if z!=y then raise "error in test, removing accents from '#{x}' gives '#{z}', expected '#{y}'" end
+    }
+    #----
+    # sorting:
+    tests = [
+      [ "Μῆνιν ἄειδε, θεά, Πηληϊάδεω Ἀχιλῆος οὐλομένην", "ἄειδε, Ἀχιλῆος θεά, Μῆνιν οὐλομένην Πηληϊάδεω" ],
+      [ "ὠμοφάγος ᾍδης", "ᾍδης ὠμοφάγος" ],
+    ]
+    tests.each { |t|
+      x,y = t
+      a = x.split(/\s+/)
+      a = Tinycus.alpha_sort(a)
+      z = a.join(' ')
+      if z!=y then raise "error in test, sorting words on #{x} gives #{z}, expected #{y}" end
+    }
+    #----
+    filename = "test_input/bad_combining_character.txt"
+    s = nil
+    File.open(filename,'r') { |f|
+      s = f.gets(nil) # nil means read whole file
+    }
+    if s.nil? || s.length<10 then raise "wtf?" end
+    s2 = Cleanup.clean_up_greek_combining_characters(s,allow_latin:true) # allow_latin is so we can put in stuff like xml fragments
+    if s2==s then raise "error in test, no change with Cleanup.clean_up_greek_combining_characters" end
+    Cleanup.clean_up_grotty_greek_one_word(s,allow_latin:true)
+    #----
+    # Beta code/unicode conversion. In the following, there can actually be more than one beta code representation
+    # of the same unicode character, so if the third-element flag is true, we don't expect uni to beta to work.
+    [
+      ["ἐν","E)N"],
+      ["Ἄλφα","*)/alfa"], # breathing and tonal accent are supposed to come first for uppercase
+      ["προϊέναι","PROI+E/NAI"],
+      ["πρός","pro/j"],
+      ["πρός","pro/s",true],
+      ["πρός","pro/s2",true],
+      ["σῖτον","si=ton"],
+      ["σῖτον","s1i=ton",true],
+      ["μάχαιρᾰ","MA/XAIRA'"],
+      ["μαχαίρᾱς","MAXAI/RA&j"],
+      ["ἤφυσ’","h)/fus'"],
+      ["·—",":_"],
+      ["ἴκτινος","i/)ktinos",true], # probably should be )/, but we should deal with it anyway
+      ["ἴκτινος","i)/ktinos",true],
+      ["βοΐ","boi/+"],
+      ["βοΐ","boi+/",true],
+      ["ᾇ","a(=|"],
+      ["ᾇτε","a(=|te"],
+      #["",""],
+    ].each { |t|
+      unicode,beta_code,no_uni_to_beta = t
+      beta_code = beta_code.downcase
+      if no_uni_to_beta.nil? then no_uni_to_beta=false end
+      if !no_uni_to_beta && Tinycus.greek_unicode_to_beta_code(unicode)!=beta_code then
+        raise "error in Tinycus.greek_unicode_to_beta_code, #{unicode} converts to #{Tinycus.greek_unicode_to_beta_code(unicode)}, expected #{beta_code}"
+      end
+      if Tinycus.greek_beta_code_to_unicode(beta_code)!=unicode then
+        raise "error in Tinycus.greek_beta_code_to_unicode, #{beta_code} converts to #{Tinycus.greek_beta_code_to_unicode(beta_code)}, expected #{unicode}"
+      end
+    }
+
+    [
+      ["ἀντῑκρύ̄","ἀντι_κρύ_"],
+      ["ἀνῑ̆άζω","ἀνιάζω"],  # iota with macron, followed by a combining breve, i.e., can be either short or long
+      ["ἀ̄ϊ̄κή","ἀ_ϊ_κή"],  # iota with diaresis, followed by a combining macron
+    ].each { |x|
+      a,b = x
+      b_actual = Tinycus::Tr.macronized_to_underbar_style(a)
+      if b_actual==b then
+        print "passed test of macronized_to_underbar_style, #{a} -> #{b}\n"
+      else
+        raise "failed, #{x}, actual = #{b_actual}"
+      end
+    }
+
+
+  end
+
+  # fixme:
+  #   Determine byte order and make sure we convert to native (i.e., BE rather than LE if we're on a big-endian machine).
+  @@bloater = Encoding::Converter.new('UTF-8','UTF-32LE')
+  @@shrinker = Encoding::Converter.new('UTF-32LE','UTF-8')
+
+  def Tinycus.bloat(s) # private method
+    return @@bloater.convert(s)
+  end
+  def Tinycus.shrink(s) # private method
+    return @@shrinker.convert(s)
+  end
+
+  class Tinycus::MiscGreek
+    def MiscGreek.run_tests
+      print "testing MiscGreek.add_second_accent...\n"
+      [['θεμείλια','θεμείλιά'],
+       ['πόλεμονδε','πόλεμόνδε'],
+       ['οἶκονδε','οἶκόνδε'],
+       ['τῆσδε','τῆσδέ']
+      ].each { |x|
+        single,two = x
+        y = MiscGreek.add_second_accent(single)
+        print "  #{single} #{two} #{y}\n"
+        if y!=two then raise "expected #{two}, got #{y}" end
+      }
+    end
+
+    def MiscGreek.estimate_syll_count(x)
+      # A rough approximation, for cases where we don't need perfect precision and either don't have Ransom's greek/syllab.rb
+      # or don't want the performance hit.
+      x = x.downcase
+      x = x.gsub(/[ϊ]/,'e') # prevent it from being misinterpreted as a diphthong after the diaresis is stripped below
+      x = Tr.remove_accents_from_greek(x)
+      x = x.gsub(/(αι|ει|οι|ου)/,'e')
+      return x.scan(/[αειουηωe]/).length
+    end
+
+    def MiscGreek.has_acute(x)
+      return Tr.remove_acute_from_greek(x)!=x
+    end
+
+    def MiscGreek.has_grave(x)
+      return Tr.remove_grave_from_greek(x)!=x
+    end
+
+    def MiscGreek.has_circumflex(x)
+      return Tr.remove_circumflex_from_greek(x)!=x
+    end
+
+    def MiscGreek.has_tonal_accent(x)
+      return Tr.remove_tonal_accents_from_greek(x)!=x
+    end
+
+    def MiscGreek.add_second_accent(w_orig)
+      # e.g., if w is θεμείλια, returns θεμείλιά
+      w = w_orig.clone # shallow copy, works on a string; I'm not clear in why this is necessary, but it is; modification to w_orig is visible in
+                       # output of make test_misc_greek
+      if w=~/(.*)δε$/ then
+        stem = $1
+        nsyll = MiscGreek.estimate_syll_count(stem)
+        has_circumflex = MiscGreek.has_circumflex(stem)
+        if nsyll>=3 || (has_circumflex && nsyll>=2) then return MiscGreek.add_second_accent(stem)+"δε" end
+      end
+      # Locate the final vowel:
+      x = Tr.remove_accents_from_greek(w).downcase
+      (x.length-1).downto(1) { |i|
+        if x[i]=~/[αειουηω]/ then
+          w[i] = Tr.add_acute_to_greek(Tr.remove_acute_and_grave_from_greek(w[i]))
+          return w
+        end
+      }
+      return w
+    end
+
+    def MiscGreek.all_cases_and_accents(w)
+      # For a given word, try to predict every possible form it could take in a text, including
+      # both possible capitalizations, acute/grave, and multiple accents.
+      # The word w should already have been converted into a canonical dictionary form (typically a single acute accent).
+      # This is not 100% perfect, mainly because the rules for multiple accents are complicated and Tinycus doesn't include a full
+      # syllabification algorithm.
+      # I tested this as a round-trip on all multiply accented words occurring in Homer. The following three words were the only
+      # ones where it failed: κάλλίον, σταφύλῇ, ὕπὸ.
+      forms = [w.downcase]
+      forms = forms+forms.map { |x| Tr.greek_acute_to_grave(x) }
+      forms = forms+forms.map { |x| MiscGreek.add_second_accent(x) }
+      forms = forms+forms.map { |x| x.capitalize }
+      return forms.uniq
+    end
+  end
+
+  class Tinycus::Tr
+
+    @@prep_remove_acute_and_grave_from_greek = nil
+    @@prep_remove_circumflex_from_greek = nil
+    @@prep_add_circumflex_to_greek = nil
+    @@prep_remove_grave_from_greek = nil
+    @@prep_remove_acute_from_greek = nil
+    @@prep_add_grave_to_greek = nil
+    @@prep_add_acute_to_greek = nil
+    @@prep_remove_diar_from_greek = nil
+    @@prep_remove_breathing_from_greek = nil
+    @@prep_add_diar_to_greek = nil
+    @@prep_remove_accents_from_greek = nil
+    @@greek_grave_to_acute = nil
+    @@greek_acute_to_grave = nil
+    @@prep_greek_to_collation_form = nil
+
+    def initialize(a,b)
+      # Initialize a data structure that represents an action equivalent to string.tr(a,b), but faster.
+      # Including redudant characters is harmless and is fixed in this constructor; it does not cause
+      # any performance hit when the object is actually used.
+      if a.length!=b.length then raise "lengths unequal, #{a.length} and #{b.length}" end
+      @l = a.length
+      @orig_tables = [a.clone,b.clone] # stash them away for testing purposes
+
+      @h = {}
+      0.upto(@l-1) { |i|
+        p,q = a[i],b[i]
+        @h[p] = q
+      }
+      @h.freeze
+      
+    end
+
+    attr_reader :l,:a,:b
+
+    def apply(s,n:false)
+      # This function tends to be a bottleneck for performance in real-world applications. I tried several algorithms.
+      # See notes in comments at top of scripts/benchmark.rb.
+
+      if n then s = s.unicode_normalize(:nfc) end # 30% performance hit, not necessary if input has already been normalized
+
+      result = ''
+      0.upto(s.length-1) { |i|
+        p = s[i]
+        q = @h[p]
+        if q.nil? then c=p else c=q end
+        result += c
+      }
+      return result
+
+    end
+
+    def self_test(alphabet)
+      # Raises an exception if it fails. Otherwise just returns silently.
+      # If not nil, then the alphabet parameter gives a list of characters that are allowed to exist in the output.
+      a,b = @orig_tables
+      if self.apply(a)!=b then raise "error in self_test, applying me to original a does not give original b" end
+      if self.apply(b)!=b then raise "error in self_test, applying me to original b does not give original b" end
+      if self.apply(self.apply(a))!=b then raise "error in self_test, fails idempotency" end
+      if !alphabet.nil? then
+        s = a+alphabet
+        ss = self.apply(s)
+        unless ss=~/^[#{alphabet}]+$/ then
+          raise "error in self_test, applying me to #{s} gives #{ss}, which contains characters not in the alphabet #{alphabet}"
+        end
+      end
+    end
+
+    def Tr.greek_grave_to_acute(s,n:false)
+      if @@greek_grave_to_acute.nil? then
+        @@greek_grave_to_acute = Tinycus::Tr.new(
+          "ÀÈÌÒÙàèìòùἂἃἊἒἓἢἣἫἲἳὂὃὊὋὓὢὣὫὰὲὴὶὸὺὼῒῢῸῂ","ÁÉÍÓÚáéíóúἄἅἌἔἕἤἥἭἴἵὄὅὌὍὕὤὥὭάέήίόύώΐΰΌῄ"
+        )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@greek_grave_to_acute.apply(s)
+    end
+
+    def Tr.greek_acute_to_grave(s,n:false)
+      if @@greek_acute_to_grave.nil? then
+        @@greek_acute_to_grave = Tinycus::Tr.new(
+          "ÁÉÍÓÚáéíóúἄἅἌἔἕἤἥἭἴἵὄὅὌὍὕὤὥὭάέήίόύώΐΰΌ","ÀÈÌÒÙàèìòùἂἃἊἒἓἢἣἫἲἳὂὃὊὋὓὢὣὫὰὲὴὶὸὺὼῒῢῸ"
+        )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@greek_acute_to_grave.apply(s)
+    end
+
+    def Tr.remove_tonal_accents_from_greek(s,n:false)
+      # to do: make this more efficient by creating a single-pass tr
+      return Tr.remove_circumflex_from_greek(Tr.remove_acute_and_grave_from_greek(s,n:n))
+    end
+
+    def Tr.remove_acute_and_grave_from_greek(s,n:false)
+      if @@prep_remove_acute_and_grave_from_greek.nil? then
+        @@prep_remove_acute_and_grave_from_greek =       
+            Tinycus::Tr.new(
+              "ÀÁàáÈÉèéÌÍìíÒÓòóÙÚùúÝýΆάἂἃἄἅἊἌἍὰᾴΈέἒἓἔἕἜἝὲήἢἣἤἥἫἬἭὴᾓᾔᾕῂῄΊΐίἲἳἴἵἼἽὶῒΌόὂὃὄὅὊὋὌὍὸῸΰύὓὔὕὝὺῢώὢὣὤὥὫὬὭὼᾤῴᾍ",
+              "AAaaEEeeIIiiOOooUUuuYyΑαἀἁἀἁἈἈἉαᾳΕεἐἑἐἑἘἙεηἠἡἠἡἩἨἩηᾑᾐᾑῃῃΙϊιἰἱἰἱἸἹιϊΟοὀὁὀὁὈὉὈὉοΟϋυὑὐὑὙυϋωὠὡὠὡὩὨὩωᾠῳᾉ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_acute_and_grave_from_greek.apply(s)
+    end
+
+    def Tr.remove_circumflex_from_greek(s,n:false)
+      if @@prep_remove_circumflex_from_greek.nil? then
+        @@prep_remove_circumflex_from_greek =       
+            Tinycus::Tr.new(
+              "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿΆΈΊΌΐάέήίΰϊϋόύώỏἀἁἂἃἄἅἆἈἉἊἌἍἎἐἑἒἓἔἕἘἙἜἝἠἡἢἣἤἥἦἧἨἩἫἬἭἮἯἰἱἲἳἴἵἶἷἸἹἼἽἾὀὁὂὃὄὅὈὉὊὋὌὍὐὑὓὔὕὖὗὙὝὠὡὢὣὤὥὦὧὨὩὫὬὭὮὯὰὲὴὶὸὺὼᾐᾑᾓᾔᾕᾖᾗᾠᾤᾦᾧᾰᾱᾳᾴᾶᾷᾸᾹῂῃῄῆῇῐῑῒῖῗῘῙῠῡῢῥῦῨῩῬῳῴῶῷῸ",
+              "ÀÁAAÄÅÆÇÈÉEËÌÍIÏNÒÓOOÖØÙÚUÜÝàáaaäåæçèéeëìíiïnòóooöøùúuüýÿΆΈΊΌΐάέήίΰϊϋόύώỏἀἁἂἃἄἅἀἈἉἊἌἍἈἐἑἒἓἔἕἘἙἜἝἠἡἢἣἤἥἠἡἨἩἫἬἭἨἩἰἱἲἳἴἵἰἱἸἹἼἽἸὀὁὂὃὄὅὈὉὊὋὌὍὐὑὓὔὕὐὑὙὝὠὡὢὣὤὥὠὡὨὩὫὬὭὨὩὰὲὴὶὸὺὼᾐᾑᾓᾔᾕᾐᾑᾠᾤᾠᾡᾰᾱᾳᾴαᾳᾸᾹῂῃῄηῃῐῑῒιϊῘῙῠῡῢῥυῨῩῬῳῴωῳῸ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_circumflex_from_greek.apply(s)
+    end
+
+    def Tr.add_circumflex_to_greek(s,n:false)
+      if @@prep_add_circumflex_to_greek.nil? then
+        @@prep_add_circumflex_to_greek =       
+            Tinycus::Tr.new(
+              "AAEINOOUaaeinoouἀἈἠἡἨἩἰἱἸὐὑὠὡὨὩᾐᾑᾠᾡαᾳηῃιϊυωῳ",
+              "ÂÃÊÎÑÔÕÛâãêîñôõûἆἎἦἧἮἯἶἷἾὖὗὦὧὮὯᾖᾗᾦᾧᾶᾷῆῇῖῗῦῶῷ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_add_circumflex_to_greek.apply(s)
+    end
+
+    def Tr.remove_acute_from_greek(s,n:false)
+      if @@prep_remove_acute_from_greek.nil? then
+        @@prep_remove_acute_from_greek =       
+            Tinycus::Tr.new(
+              "ÁÉÍÓÚáéíóúἄἅἌἔἕἤἥἭἴἵὄὅὌὍὕὤὥὭάέήίόύώῄῴΐΰΌ",
+              "AEIOUaeiouἀἁἈἐἑἠἡἩἰἱὀὁὈὉὑὠὡὩαεηιουωῃῳϊϋΟ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_acute_from_greek.apply(s)
+    end
+
+    def Tr.remove_grave_from_greek(s,n:false)
+      if @@prep_remove_grave_from_greek.nil? then
+        @@prep_remove_grave_from_greek =       
+            Tinycus::Tr.new(
+              "ÀÈÌÒÙàèìòùἂἃἊἒἓἢἣἫἲἳὂὃὊὋὓὢὣὫὰὲὴὶὸὺὼῂῒῢῸ",
+              "AEIOUaeiouἀἁἈἐἑἠἡἩἰἱὀὁὈὉὑὠὡὩαεηιουωῃϊϋΟ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_grave_from_greek.apply(s)
+    end
+
+    def Tr.add_grave_to_greek(s,n:false)
+      if @@prep_add_grave_to_greek.nil? then
+        @@prep_add_grave_to_greek =       
+            Tinycus::Tr.new(
+              "AEIOUaeiouἀἁἈἐἑἠἡἩἰἱὀὁὈὉὑὠὡὩαεηιουωῃϊϋΟ",
+              "ÀÈÌÒÙàèìòùἂἃἊἒἓἢἣἫἲἳὂὃὊὋὓὢὣὫὰὲὴὶὸὺὼῂῒῢῸ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_add_grave_to_greek.apply(s)
+    end
+
+    def Tr.add_acute_to_greek(s,n:false)
+      if @@prep_add_acute_to_greek.nil? then
+        @@prep_add_acute_to_greek =       
+            Tinycus::Tr.new(
+              "AEIOUaeiouἀἁἈἐἑἠἡἩἰἱὀὁὈὉὑὠὡὩαεηιουωῃῳϊϋΟ",
+              "ÁÉÍÓÚáéíóúἄἅἌἔἕἤἥἭἴἵὄὅὌὍὕὤὥὭάέήίόύώῄῴΐΰΌ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_add_acute_to_greek.apply(s)
+    end
+
+    def Tr.remove_diar_from_greek(s,n:false)
+      # Can't do caps with diaresis, since they only exist as combining characters.
+      if @@prep_remove_diar_from_greek.nil? then
+        @@prep_remove_diar_from_greek =       
+            Tinycus::Tr.new(
+              "ϊΐῒῗϋΰῢ",
+              "ιίὶῖυύὺ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_diar_from_greek.apply(s)
+    end
+
+    def Tr.add_diar_to_greek(s,n:false)
+      # Can't do caps with diaresis, since they only exist as combining characters.
+      if @@prep_add_diar_to_greek.nil? then
+        @@prep_add_diar_to_greek =       
+            Tinycus::Tr.new(
+              "ιίὶῖυύὺ",
+              "ϊΐῒῗϋΰῢ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_add_diar_to_greek.apply(s)
+    end
+
+    def Tr.remove_breathing_from_greek(s,n:false)
+      if @@prep_remove_breathing_from_greek.nil? then
+        @@prep_remove_breathing_from_greek =       
+            Tinycus::Tr.new(
+              "ἄἌἈἈἀἈἁἉἂἊἅἍἆἉἉἃἋᾇἔἜἑἙἐἘἕἝἘἘἙἙἓἛἒἚἣἫἡἩἠἨἦἥἭἢἪἤἬᾔἬΙἧᾗἨἨᾕἭΙἩἩᾐἨΙᾖᾑἩΙἰἸἱἹἴἼἶἳἻἵἽἷἸἸἲἺἹἹὃὋὄὌὀὈὈὈὅὍὁὉὉὉῥῬῬῬῤὐὗὕὝὑὙὖὔὓὛὙὙὥὭὣὫὤὬᾤὬΙὠὨὦᾧὡὩὧᾠὨΙὨὨὢὪᾦὩὩ",
+              "άΆΑΑαΑαΑὰᾺάΆᾶΑΑὰᾺᾷέΈεΕεΕέΈΕΕΕΕὲῈὲῈὴῊηΗηΗῆήΉὴῊήΉῄΉΙῆῇΗΗῄΉΙΗΗῃΗΙῇῃΗΙιΙιΙίΊῖὶῚίΊῖΙΙὶῚΙΙὸῸόΌοΟΟΟόΌοΟΟΟρΡΡΡρυῦύΎυΥῦύὺῪΥΥώΏὼῺώΏῴΏΙωΩῶῷωΩῶῳΩΙΩΩὼῺῷΩΩ"
+            )
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_breathing_from_greek.apply(s)
+    end
+
+    def Tr.remove_accents_from_greek(s,n:false)
+      if @@prep_remove_accents_from_greek.nil? then
+        @@prep_remove_accents_from_greek = Tr.remove_accents('el')
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_remove_accents_from_greek.apply(s)
+    end
+
+    def Tr.remove_macrons_and_breves(s)
+      # This can't be implemented using my fast method, because most of these are composed characters.
+      if !(s.kind_of?(String)) then return s end
+      # ...convenience feature for stuff like parsing json data, which may include integers. Won't work for arrays containing strings.
+      # ---
+      # Letters with both a macron and a breve. The ruby script to generate this is in the commented out code below.
+      if s=~/ᾱ̆|ῑ̆|ῡ̆|Ᾱ̆|Ῑ̆|Ῡ̆/ then
+        s = s.gsub(/ᾱ̆/,'α')
+        s = s.gsub(/ῑ̆/,'ι')
+        s = s.gsub(/ῡ̆/,'υ')
+        s = s.gsub(/Ᾱ̆/,'Α')
+        s = s.gsub(/Ῑ̆/,'Ι')
+        s = s.gsub(/Ῡ̆/,'Υ')
+      end
+      # ---
+      s = s.gsub(/ϊ̄/,'ϊ') # iota with macron and diaresis; happens in Cunliffe for ἀϊκη, ὠΐετο, ὁμοιΐου, ὁμοίϊος
+      s = s.gsub(/ΐ̄/,'ΐ') # iota with macron, diaresis, and acute; happens in Cunliffe for ὀΐομαι
+      # ---
+      # The following  is kludgy, probably not the right way to handle this. Should this just be in the loop at the end?
+      s = s.gsub(/ῐ/,"ῐ")
+      # ῐ = iota + combining breve
+      # ῐ = iota with vrachy
+      s = s.gsub(/ῑ/,"ι")
+      s = s.gsub(/ᾰ/,"α")
+      s = s.gsub(/ᾱ̆/,"α") # alpha with both a macron and a breve!?
+      # ---
+      s = Tinycus::Util.safe_normalize(s)
+      s = s.tr("āēīōūӯ","aeiouy") # latin
+      s = s.tr("ᾰᾱᾸᾹῐῑῘῙῠῡῨῩ","ααΑΑιιΙΙυυΥΥ")
+      # Accent combined with macron. The monospaced fonts I'm using for coding display these incorrectly, and I also don't know how to type them.
+      #         Furthermore, these seem to be represented as multiple characters, so that tr won't work. The following will be slow on short strings,
+      #         but should perform well on long ones.
+      #         The following isn't really an exhaustive list of vowels.
+      "άίύὰὶὺΆΊΎᾺῚῪἀἐἰὀὐἠὠἁἑἱὁὑἡὡἄἔἴὄὔἤὤἂἒἲὂὒἢὢἅἕἵὅὕἥὥἃἓἳὃὓἣὣΐῒ".chars.each { |c|
+        [772,774].each { |combining| # 772=combining macron, 774=combining breve (773=combining overline, presumably used for math)
+          m = [c.ord, combining].pack("U*") # is not a single character
+          s = s.gsub(/#{m}/,c)
+        }
+      }
+      # ---
+      s = s.unicode_normalize(:nfc) # found empirically that this was necessary, don't remove
+      # ---
+      return s
+=begin
+--------------------------------------------------------
+m = [0x1fb1,0x1FD1,0x1fe1]
+a = m.map { |u| [u, 0x0306].pack('U*') }
+a = a + a.map { |s| s.upcase }
+i=0
+a.each { |c|
+  cc = ['α','ι','υ','Α','Ι','Υ'][i]
+  i += 1
+  print "    s = s.gsub(/#{c}/,'#{cc}')\n"
+}
+--------------------------------------------------------
+=end
+end
+
+    def Tr.macronized_to_underbar_style(s)
+      # Changes a macronized string to one that looks like this: ἕννυ_μι.
+      # The lists in the regexes are generated by the commented-out scripts below, and are not actually totally comprehensive. 
+      # We don't handle grave and circumflex accents, but those don't occur in dictionary headers with macrons.
+      # For an IfMows object in Ifthimos, this can be done using stringify(macronization:'underbar').
+      # ---
+      x = s.clone
+      x = x.gsub(/(ϊ̄)/) { "#{Tinycus::Tr.remove_macrons_and_breves($1)}_" } # iota with diaresis and macron, occurs in ἀϊκή
+      # First handle letters that have both a macron and a breve, treating them as if they weren't macronized at all:
+      x = x.gsub(/(ᾱ̆|ῑ̆|ῡ̆|Ᾱ̆|Ῑ̆|Ῡ̆)/) { Tinycus::Tr.remove_macrons_and_breves($1) }
+      # Next handle the ones that have macrons only:
+      x = x.gsub(/(ᾱ|ῑ|ῡ|ά̄|ί̄|ύ̄|ἀ̄|ἁ̄|ἄ̄|ἅ̄|ἰ̄|ἱ̄|ἴ̄|ἵ̄|ὐ̄|ὑ̄|ὔ̄|ὕ̄|Ᾱ|Ῑ|Ῡ|Ά̄|Ί̄|Ύ̄|Ἀ̄|Ἁ̄|Ἄ̄|Ἅ̄|Ἰ̄|Ἱ̄|Ἴ̄|Ἵ̄|Υ̓̄|Ὑ̄|Υ̓́̄|Ὕ̄)/) { "#{Tinycus::Tr.remove_macrons_and_breves($1)}_" }
+      # Finally, remove all breves:
+      return Tinycus::Tr.remove_macrons_and_breves(x)
+=begin
+------------------------------------
+m = [0x1fb1,0x1FD1,0x1fe1]
+a = m.map { |u| [u, 0x0306].pack('U*') }
+a = a + a.map { |s| s.upcase }
+print a.join('|'),"--\n"
+------------------------------------
+a = ['ᾱ','ῑ','ῡ']
+[
+  0x03ac,0x03af,0x03cd
+].each { |i|
+  x = [i, 0x0304].pack('U*')
+  a.push(x)
+}
+[
+  0x1f00,0x1f30,0x1f50
+].each { |i|
+  [0,1,4,5].each { |j|
+    x = [i+j, 0x0304].pack('U*')
+    a.push(x)
+  }
+}
+a = a + a.map { |s| s.upcase }
+print a.join('|'),"--\n"
+------------------------------------
+=end
+    end
+
+    def Tr.get_greek_collation_tr
+      if @@prep_greek_to_collation_form.nil? then
+        @@prep_greek_to_collation_form = Tr.collation_form('el')
+      end
+      return @@prep_greek_to_collation_form
+    end
+
+    def Tr.greek_to_collation_form(s,n:false)
+      if @@prep_greek_to_collation_form.nil? then
+        @@prep_greek_to_collation_form = Tr.collation_form('el')
+      end
+      if n then s=s.unicode_normalize(:nfc) end
+      return @@prep_greek_to_collation_form.apply(s)
+    end
+
+    def Tr.remove_accents(locale)
+      # Returns a Tinycus::Tr object which can then be used to act on strings using the apply() method.
+      # The 'el' locale is a standard thing that software like ICU uses for polytonic Greek. The object constructed with this
+      # locale will also remove most accents and macrons from Latin characters, but will miss some cases like Czech, and will not handle Cyrillic.
+      t = {
+        "el"=>[
+          "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿΆΈΊΌΐάέήίΰϊϋόύώỏἀἁἂἃἄἅἆἈἉἊἌἍἎἐἑἒἓἔἕἘἙἜἝἠἡἢἣἤἥἦἧἨἩἫἬἭἮἯἰἱἲἳἴἵἶἷἸἹἼἽἾὀὁὂὃὄὅὈὉὊὋὌὍὐὑὓὔὕὖὗὙὝὠὡὢὣὤὥὦὧὨὩὫὬὭὮὯὰὲὴὶὸὺὼᾐᾑᾓᾔᾕᾖᾗᾠᾤᾦᾧᾰᾱᾳᾴᾶᾷᾸᾹῂῃῄῆῇῐῑῒῖῗῘῙῠῡῢῥῦῨῩῬῳῴῶῷῸῤᾆᾄᾂᾁᾇᾅᾃᾍᾡ",
+          "AAAAAAÆCEEEEIIIINOOOOOOUUUUYaaaaaaæceeeeiiiinoooooouuuuyyΑΕΙΟιαεηιυιυουωoαααααααΑΑΑΑΑΑεεεεεεΕΕΕΕηηηηηηηηΗΗΗΗΗΗΗιιιιιιιιΙΙΙΙΙοοοοοοΟΟΟΟΟΟυυυυυυυΥΥωωωωωωωωΩΩΩΩΩΩΩαεηιουωηηηηηηηωωωωααααααΑΑηηηηηιιιιιΙΙυυυρυΥΥΡωωωωΟραααααααΑω"
+        ]
+      }
+      tables = t[locale]
+      if tables.nil? then raise "unknown locale: #{locale}, known locales are: #{t.keys.join(' ')}" end
+      result = Tinycus::Tr.new(tables[0],tables[1])
+      return result
+    end
+
+    def Tr.collation_form(locale)
+      # Returns a Tinycus::Tr object which can then be used to act on strings using the apply() method. Gives a form that 
+      # can be alphabetized properly.
+      # The 'el' locale is a standard thing that software like ICU uses for polytonic Greek. The object constructed with this
+      # locale will also produce correct results for most Latin-script words, will miss some cases like Czech, and will not handle Cyrillic.
+      t = {
+        "el"=>[
+          "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿΆΈΊΌΐάέήίΰϊϋόύώỏἀἁἂἃἄἅἆἈἉἊἌἍἎἐἑἒἓἔἕἘἙἜἝἠἡἢἣἤἥἦἧἨἩἫἬἭἮἯἰἱἲἳἴἵἶἷἸἹἼἽἾὀὁὂὃὄὅὈὉὊὋὌὍὐὑὓὔὕὖὗὙὝὠὡὢὣὤὥὦὧὨὩὫὬὭὮὯὰὲὴὶὸὺὼᾐᾑᾓᾔᾕᾖᾗᾠᾤᾦᾧᾰᾱᾳᾴᾶᾷᾸᾹῂῃῄῆῇῐῑῒῖῗῘῙῠῡῢῥῦῨῩῬῳῴῶῷῸῤᾆᾄᾂᾁᾇᾅᾃΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩςᾍ",
+          "aaaaaaæceeeeiiiinoooooouuuuyaaaaaaæceeeeiiiinoooooouuuuyyαειοιαεηιυιυουωoαααααααααααααεεεεεεεεεεηηηηηηηηηηηηηηηιιιιιιιιιιιιιοοοοοοοοοοοουυυυυυυυυωωωωωωωωωωωωωωωαεηιουωηηηηηηηωωωωααααααααηηηηηιιιιιιιυυυρυυυρωωωωορααααααααβγδεζηθικλμνξοπρστυφχψωσΑ"
+        ]
+      }
+      tables = t[locale]
+      if tables.nil? then raise "unknown locale: #{locale}, known locales are: #{t.keys.join(' ')}" end
+      result = Tinycus::Tr.new(tables[0],tables[1])
+      return result
+    end
+
+    def Tr.add_breathing_to_character(c,what)
+      plain,d = Tinycus.disassemble_greek_char(c)
+      d['breathing'] = what
+      return Tinycus.assemble_greek_char(plain,d)
+    end
+
+
+
+    def Tr.run_tests
+      # to execute this, do a "make test_tr"
+      ['el'].each { |locale|
+        tr = Tinycus::Tr.remove_accents(locale)
+        if locale=='el' then alphabet=Script.alphabet('greek')+Script.alphabet('latin')+'Ææ' else alphabet=nil end
+        tr.self_test(alphabet)
+        print "Passed self-test on locale #{locale}.\n"
+      }
+    end
+
+
+  end # class Tr
+
+  class Tinycus::Script
+
+    def Script.alphabet(script,c:'both')
+      # Script can be 'latin', 'greek', or 'hebrew'.
+      # C can be both, lowercase, or uppercase.
+      # For scripts that don't have case, c is ignored.
+      t = {
+        'latin'=>{'has_case'=>true},
+        'greek'=>{'has_case'=>true},
+        'hebrew'=>{'has_case'=>false},
+      }
+      data = t[script]
+      if data.nil? then raise "unknown script: #{script}, possible values are: #{t.keys.join(' ')}" end
+      has_case = data['has_case']
+      if !has_case then return Script.alphabet_helper(script,nil) end
+      if c=='both' then return Script.alphabet(script,c:"lowercase")+Script.alphabet(script,c:"uppercase") end
+      # If we fall through to here, then we're doing a single case of an alphabet that has two cases.
+      if c=='lowercase' then return Script.alphabet_helper(script,true) end
+      if c=='uppercase' then return Script.alphabet_helper(script,false).upcase end
+      die("illegal value of c=#{c}, must be both, lowercase, or uppercase")
+    end
+
+    def Script.alphabet_helper(script,include_lc_only_chars)
+      if script=='latin'  then return 'abcdefghijklmnopqrstuvwxyz' end
+      if script=='greek'  then 
+        result = 'αβγδεζηθικλμνξοπρστυφχψω'
+        if include_lc_only_chars then result = result+'ς' end
+        return result.unicode_normalize(:nfc)
+      end
+      if self.name=='hebrew'  then return 'אבגדהוזחטילמנסעפצקרשתםןףץ'.unicode_normalize(:nfc) end
+      # ... Word-final forms are all at the end.
+      #     To edit the Hebrew list, use mg, not emacs. Emacs tries to be smart about RTL but freaks out and gets it
+      #      wrong on a line that mixes RTL and LTR.
+      raise "no alphabet available for script #{script}"
+    end
+
+  end
+
+  class Tinycus::Cleanup
+
+    def Cleanup.clean_up_grotty_greek(s,allow_latin:false,clean_perseus:true,standardize_punctuation:true)
+      # Designed for external data sources that can have all kinds of nasty crap in them. Slow, thorough, silent, and brutal.
+      a = s.split(/(\s+)/) # returns a string in which even indices are words, odd indices are whitespace
+      b = []
+      0.upto(a.length-1) { |i|
+        w = a[i]
+        if i%2==0 then
+          looks_greek = true
+          if w=~/[a-zA-Z]/ then looks_greek=false end # for speed and reliability; if it contains Latin letters, it shouldn't be a greek word
+          if looks_greek && !(w=~/[α-ωΑ-Ως]/) && \
+                          !(Tinycus::Tr.remove_accents_from_greek(w.unicode_normalize(:nfc))=~/[α-ωΑ-Ως]/) then looks_greek=false end
+          if looks_greek then
+            w=Cleanup.clean_up_grotty_greek_one_word(w,allow_latin:allow_latin,clean_perseus:clean_perseus,standardize_punctuation:standardize_punctuation)
+          end
+        end
+        b.push(w)
+      }
+      s = b.join('')
+      if standardize_punctuation then
+        s = Cleanup.standardize_greek_punctuation(s)
+      end
+      s = s.unicode_normalize(:nfc) # does stuff like changing deprecated 8059 (upsilon with oxia) to 973 (upsilon with tonos)
+      return s
+    end
+
+    def Cleanup.clean_up_grotty_greek_one_word(s,allow_latin:false,clean_perseus:true,standardize_punctuation:true)
+      # This works on a single word.
+      s = s.unicode_normalize(:nfc)
+      s = Cleanup.clean_up_greek_combining_characters(s,allow_latin:allow_latin)
+      # In Perseus's Polybius, they have bracketed text sometimes. In their system, this should probably be a separate punctuation token.
+      if clean_perseus then
+        s = s.sub(/\[/,'') # Hesiod, δεσμὸ]ν; also occurs sometimes like [word]
+        s = s.sub(/\]/,'')
+        s = s.sub(/^\(/,'')
+        s = s.sub(/\)$/,'')
+        s = s.sub(/\}/,'') # Hesiod, Ι{ππώ
+        s = s.sub(/\{/,'')
+        s = s.sub(/&?απο[σς];/,"᾽") # software bug in perseus, is '&apos;' transliterated into Greek
+        s.sub!(/\-$/,"᾽") # e.g., in Thucydides, Perseus has δοκεῖ δέ μοι, οὐδὲ τοὔνομα with τοὔνομα written as τ- οὔνομα
+        s.sub!(/\-/,'') # e.g., in Thucydides, Perseus has ἀντίσχουσαν lemmatized as ἀντί-ἴσχω
+      end
+      s = s.sub(/σ$/,'ς') # this won't work if there's trailing punct; what is the right way to spell a word that ends with a sigma, then an elision mark?
+      s = Cleanup.clean_up_greek_beta_code(s)
+      greek_koronis = [8125].pack('U')
+      if s[0]==greek_koronis then
+        s = s[1..-1] # this happens in perseus for the lemma ἀθήνη, which they have encoded as 787 7936 952 ..., i.e., the
+        #             breathing mark is there twice, once as a combining comma above and once as part of the composed character ἀ
+        #             https://github.com/PerseusDL/treebank_data/issues/37
+      end
+      s = Cleanup.standardize_greek_punctuation(s) if standardize_punctuation
+      if !allow_latin && s=~/[^[:alpha:]᾽[0-9]\?;,.··«»’᾽—“”]/ then raise "word #{s} contains unexpected characters; unicode=#{s.chars.map { |x| x.ord}}" end
+      return s
+    end
+
+    def Cleanup.clean_up_greek_combining_characters(s,allow_latin:false)
+      combining_comma_above = [787].pack('U')
+      combining_acute_accent = [769].pack('U')
+      greek_koronis = [8125].pack('U')
+      # s = s.sub(/α#{combining_comma_above}#{combining_acute_accent}/,'ἄ') # my error introduced in Lemming; no longer needed
+      s = s.sub(/#{combining_comma_above}/,greek_koronis)
+      # ... mistaken use of combining comma above rather than the spacing version
+      #     https://github.com/PerseusDL/treebank_data/issues/31
+      s = s.sub(/#{combining_acute_accent}/,'') # occurs once in Perseus's Plutarch, in a place where it's redundant
+      s = s.sub(/#{[788].pack('U')}/,'') # combining reversed comma above; occurs once in Perseus's Polybius, where it's on a capital Ι
+      # seeming one-off errors in perseus:
+      s2 = s
+      s2 = s2.sub(/#{[8158, 7973].pack('U')}/,"ἥ") # dasia and oxia combining char with eta
+      s2 = s2.sub(/#{[8142, 7940].pack('U')}/,"ἄ") # psili and oxia combining char with alpha
+      s2 = s2.sub(/#{[8142, 7988].pack('U')}/,"ἴ")
+      s2 = s2.sub(/ἄἄ/,'ἄ') # why is this necessary...??
+      s2 = s2.sub(/ἥἥ/,'ἥ') # why is this necessary...??
+      s2 = s2.sub(/#{[769].pack('U')}([μτ])/) {$1} # accent on a mu or tau, obvious error
+      s2 = s2.sub(/#{[769].pack('U')}ε/) {'έ'}
+      s2 = s2.sub(/#{[180].pack('U')}([κ])/) {$1} # accent on a kappa, obvious error
+      s2 = s2.sub(/#{[834].pack('U')}/,'') # what the heck is this?  
+      s2 = s2.sub(/ʽ([ἁἑἱὁὑἡὡ])/) {$1} # redundant rough breathing mark
+      # another repeating error:
+      s2 = s2.sub(/(?<=[[:alpha:]][[:alpha:]])([ἀἐἰὀὐἠὠ])(?![[:alpha:]])/) { $1.tr("ἀἐἰὀὐἠὠ","αειουηω")+"᾽" }
+      # ... smooth breathing on the last character of a long word; this is a mistake in representation of elision
+      #     https://github.com/PerseusDL/treebank_data/issues/31
+      s = s2
+      return s
+    end
+
+    def Cleanup.clean_up_greek_beta_code(s)
+      # This was for when I mistakenly used old beta code version of project perseus.
+      # Even with perseus 2.1, some stuff seems to come through that looks like beta code, e.g., ἀργει~ος.
+      # https://github.com/PerseusDL/treebank_data/issues/30
+      s = s.sub(/\((.)/) { $1.tr("αειουηω","ἁἑἱὁὑἡὡ") }
+      s = s.sub(/\)(.)/) { $1.tr("αειουηω","ἀἐἰὀὐἠὠ") } 
+      s = s.sub(/(.)~/) { $1.tr("αιυηω","ᾶῖῦῆῶ") } 
+      s = s.sub(/\|/,'ϊ') 
+      s = s.sub(/\/(.)/) { $1.tr("αειουηω","άέίόύήώ") }
+      s = s.sub(/&θυοτ;/,'')
+      s = s.sub(/θεοισ=ν/,'θεοῖσιν')
+      s = s.sub(/ὀ=νοψ1/,'οἴνοπα1')
+      s = s.sub(/π=ας/,'πᾶς')
+      return s
+    end
+
+    def Cleanup.standardize_greek_punctuation(s)
+      # Works on any string, doesn't have to be a single word. Standardize elision character and middle dot/ano teleia.
+      # Perseus and Monro/Allen write ρ with breathing mark instead of ρ᾽ when there's elision:
+      s = s.gsub(/(?<=[[:alpha:]])[ῤῥ](?![[:alpha:]])/,'ρ᾽')
+      # ... Note that we do need to reinsert the breathing mark, or else we lose the info needed to do accurate lemmatization. Cf. Spelling module.
+      # Wikisource has ῤῥ in the middle of words, e.g., χείμαῤῥοι, which OCT and Perseus don't have:
+      s = s.gsub(/(?<=[[:alpha:]])ῤῥ(?=[[:alpha:]])/,'ρρ')
+      # Standardize the elision character:
+      s = s.gsub(/[᾽’'](?![[:alpha:]])/,'᾽')
+      # ... There are other possibilities (see comments in ransom/contains_greek_elision), but these should already have been taken care of in Lemming.
+      s = s.gsub(/#{[183].pack('U')}/,[903].pack('U')) # ano teleia has two forms, B7=183 and 387=903; GFS Porson and Olga only have the latter code point
+      return s
+    end
+
+  end
+
+  class Tinycus::Util
+def Util.is_valid_utf8(s)
+  return s.clone.force_encoding("UTF-8").valid_encoding?
+end
+
+def Util.explain_how_strings_differ(x,y)
+  result = []
+  result.push("comparing '#{x}' to '#{y}'")
+  if x==y then
+    result.push("  strings are equal")
+  else
+    if x.length!=y.length then
+      result.push("  strings differ in length, #{x.length} and #{y.length}")
+    else
+      0.upto(x.length-1) { |i|
+        if x[i]!=y[i] then
+          result.push("  strings differ at position #{i}, #{x[i]}!=#{y[i]}, codes are #{x[i].ord} and #{y[i].ord}")
+        end
+      }
+    end
+  end
+  return result.map { |line| line+"\n" }.join('')
+end
+
+def Util.explain_chars_in_string(s)
+  result = []
+  result.push("explaining the characters in the string '#{s}':")
+  0.upto(s.length-1) { |i|
+    result.push("  [#{i}] = '#{s[i]}', code = #{s[i].ord}")
+  }
+  return result.map { |line| line+"\n" }.join('')
+end
+
+def Util.longest_common_initial_substring(aa,bb,script)
+  if aa.length<bb.length then return longest_common_initial_substring(bb,aa,script) end
+  a = aa
+  b = bb
+  if script=='greek' then a=aa.gsub(/ς$/,'σ'); b=bb.gsub(/ς$/,'σ') end
+  # From here on, we're guaranteed that a is at least as long as b and neither has a final lunate sigma.
+  best = ''
+  0.upto(b.length-1) { |l|
+    if a[0..l]==b[0..l] then best=a[0..l] end
+  }
+  if script=='greek' then best=best.sub(/σ$/,'ς') end
+  return best
+end
+
+def Util.words(s)
+  # Splits a string into an array of words, eliminating whitespace and interword punctuation.
+  # Don't use this for making word-by-word running hashes; that's what split_string_at_whitespace() is for.
+  # Knows about apostrophe for English and two elision characters for Greek.
+  # For a better-engineered version of this, see genos.string_to_words().
+  return s.scan(/[[:alpha:]'’᾽]+/)
+end
+
+def Util.strip_whitespace(s)
+  # strips leading and trailing whitespace
+  return s.sub(/^\s+/,'').sub(/\s+$/,'')
+end
+
+def Util.split_string_at_whitespace(text)
+  # Returns an array like [['The',' '],['quick',' '],...]. Every element is guaranteed to be a two-element list.
+  # In the final pair, the whitespace will be a null string if the text doesn't end with whitespace.
+  # This is basically meant for simple, reproducible word-by-word hashing (WhereAt.auto_hash), not for
+  # human-readable text processing, so don't use it for other purposes or fiddle with it to make it work
+  # for that purpose. For human-readable extraction of words, without punctuation, see words() above.
+  a = text.split(/(\s+)/)
+  if a.length%2==1 then a.push('') end
+  b = []
+  0.upto(a.length/2-1) { |i| b.push([a[2*i],a[2*i+1]]) }
+  return b
+end
+
+def Util.split_string_into_paragraphs(text)
+  # Returns a list like ["This is a paragraph.","\n\n","Another paragraph.","\n  \n\t\n",...].
+  # Guaranteed to have even length, so final element may be a null string.
+  # Like split_string_at_whitespace(), this is meant to be used for reproducible creation of hashes.
+  paras_and_delimiters = text.split(/(\s*(?:\n[ \t]*){2,}\s*)/) # even indices=paragraphs, odd=delimiters
+  if paras_and_delimiters.length%2==1 then paras_and_delimiters.push('') end # input doesn't end with a delimiter
+  return paras_and_delimiters
+end
+
+def Util.substr(x,i,len)
+  # Basically returns x[i..(i+len-1)], but doesn't do screwy stuff in cases like i=0, len=0.
+  result = ''
+  i.upto(i+len-1) { |m|
+    result = result+x[m]
+  }
+  return result
+end
+
+def Util.texify_quotes(s)
+  s = s.gsub(/((?<=[a-zA-Z]))'(?=[a-zA-Z])/,'__ENGLISH_INTERNAL_APOSTROPHE__')
+  # We don't want [[:alpha:]], because Greek doesn't use mid-word apostrophes, and we don't want to get confused by cases where elision
+  # was marked by an ASCII apostrophe.
+  # Handle nested quotes, working from the inside out.
+  1.upto(3) { |i| # handle up to three levels
+    [[%q('),'SINGLE'],[%q("),'DOUBLE']].each { |x|
+      char,kind = x
+      s = s.gsub(/(?<![[:alpha:]])#{char}([^'"]+)#{char}(?![[:alpha:]])/) {"__OPEN_#{kind}_QUOTES__#{$1}__CLOSE_#{kind}_QUOTES__"}
+      # ... negative lookbehind and negative lookahead help to ensure we don't get confused
+    }
+  }
+  [['__OPEN_SINGLE_QUOTES__',%q(`)],    ['__CLOSE_SINGLE_QUOTES__',%q(')], 
+   ['__OPEN_DOUBLE_QUOTES__',%q(``)],   ['__CLOSE_DOUBLE_QUOTES__',%q('')], 
+   ['__ENGLISH_INTERNAL_APOSTROPHE__',%q(')]    ].each { |x|
+    marker,replace_with = x
+    s = s.gsub(/#{marker}/,replace_with)
+  }
+  return s
+end
+
+def Util.canonicalize_greek_word(w,n:false)
+  # Works on a single word, not an entire string.
+  # Is designed so that calling it on a Latin word is fast and harmless.
+  # The n argument has the same definition as in Tinycus::Tr.remove_accents_from_greek().
+  if !looks_greek(w) then return w end # is fast on Latin script
+  if n then w = w.unicode_normalize(:nfc) end
+  w = to_single_accent(w)
+  w = Tinycus::Cleanup.standardize_greek_punctuation(w)
+  return w
+end
+
+def Util.looks_greek(w,depth:0)
+  # designed to be fast
+  if w=='' then	return false end
+  if w=~/[a-zA-Z]/ then return false end
+  if w=~/[α-ωΑ-Ως]/ then return true end
+  if depth>=2 then return false end # happens if the string contains only punctuation, etc.
+  return looks_greek(Tinycus::Tr.remove_accents_from_greek(w),depth:depth+1) # slow fallback, almost never needed
+end
+
+def Util.mixes_scripts(s)
+  # not designed to be super fast or super accurate, just a quick check
+  if s=='' then	return false end
+  if s=~/[a-zA-Z]/ then has_latin=true else has_latin=false end
+  if Tinycus::Tr.remove_accents_from_greek(s)=~/[α-ωΑ-Ως]/ then has_greek=true else has_greek=false end
+  return (has_latin && has_greek)
+end
+
+def Util.to_single_accent(w,grave_to_acute:true,n:false)
+  # In most cases, it's better to use canonicalize_greek_word() rather than this.
+  # If the word has both an acute and a grave, remove the grave. If it has only a grave, change it to an acute.
+  # This is used e.g. in LemmaUtil.make_inflected_form_flavored_like_lemma.
+  # Testing: ruby -e "require './lib/string_util'; print to_single_accent('χεῖράς')"
+  # The n argument has the same definition as in Tinycus::Tr.remove_accents_from_greek().
+  if Tinycus::Tr.remove_accents_from_greek(w,n:n)==w then return w end # for efficiency
+  if Tinycus::MiscGreek.has_circumflex(w) then return Tinycus::Tr.remove_acute_and_grave_from_greek(w) end
+  acc = []
+  w_no_acute_or_grave = Tinycus::Tr.remove_acute_and_grave_from_greek(w)
+  0.upto(w.chars.length-1) { |i|
+    if w_no_acute_or_grave[i]!=w[i] then acc.push(i) end
+  }
+  if acc.length>1 then
+    # Remove every accent but the first.
+    ww = w.dup
+    1.upto(acc.length-1) { |m|
+      i = acc[m]
+      ww[i] = Tinycus::Tr.remove_acute_and_grave_from_greek(ww[i])
+    }
+    return ww
+  else
+    if grave_to_acute then
+      return Tinycus::Tr.greek_grave_to_acute(w)
+    else
+      return w
+    end
+  end
+end
+
+def Util.remove_punctuation(s)
+  # also removes whitespace
+  return s.gsub(/[^[:alpha:]]/,'')
+end
+
+def Util.safe_normalize(s)
+  begin
+    return s.encode("UTF-8").unicode_normalize(:nfc)
+  rescue
+    # is probably 8-bit ascii/ISO-8859-1?
+    return s
+  end  
+end
+
+
+def Util.lc_underbar(s)
+  return s.downcase.gsub(/ /,'_')
+end
+
+def Util.clean_up_greek(s,thorough:false,allow_latin:false,strip_punctuation:false)
+  # s is any string, can contain any script or mix of scripts, can be more than one word.
+  # Use the thorough option for external sources like raw Perseus xml files. This option is slow.
+  # The strip_punctuation option only strips punctuation that shouldn't be in a word, doesn't strip elision mark; is necessary because
+  # PROIEL Herodotus has a few errors where punct is included in word.
+  if !s.kind_of?(String) then return s end # convenience feature
+  if strip_punctuation then s=s.gsub(/[·,;«».]/,'') end
+  if thorough then
+    s = Tinycus::Cleanup.clean_up_grotty_greek(s,allow_latin:allow_latin) # standardizes punctuation by default
+  else
+    s = Tinycus::Cleanup.standardize_greek_punctuation(s) # Standardize elision character and middle dot/ano teleia.
+  end
+  return s
+end
+
+def Util.contains_greek_elision(s)
+  # The following checks are arranged in order so as to give best performance.
+  if !(s=~/[᾽’ῤῥ]/) then return false end
+  if s=~/[᾽’]/ then return true end
+  if s.length>=2 && s=~/[ῤῥ]$/ then return true end
+  return false
+  # the above are koronis (8125=14bd hex) and apostrophe (8217=2019 hex)
+  # see http://www.opoudjis.net/unicode/gkdiacritics.html
+  # Perseus sometimes has 787=313 hex, which is combining comma above, the non-spacing version of koronis. This seems
+  # to me to be a mistake on their part.
+  # https://github.com/PerseusDL/treebank_data/issues/31
+  # One could also have 700=2bc hex, spacing smooth breathing, which seems like an error, or 39=27 hex, the ascii apostrophe.
+end
+
+def Util.escape_double_quotes(s)
+  return s.gsub(/"/,'\\"') # escape double quotes
+end
+
+def Util.reverse_if_rtl(s)
+  if s=='' then return s end
+  if char_is_rtl(s[0]) then return Util.reverse_string(s) else return s end
+end
+
+def Util.reverse_string(s)
+  r = 0
+  s.chars.each { |c| r = c+r }
+  return r
+end
+
+def Util.console(*x)
+  $stderr.print *x
+end
+
+  end
+
+# The following table is output by scripts/generate_beta_code_tables.rb in Ifthimos.
+@@beta_code_conversion_json =
+<<-'JSON'
+[{"ἄ":"a)/","ἄ̄":"a)/&","ἄ̆":"a)/'","ἄ̄̆":"a)/&","ὰ":"a\\","ὰ̄":"a\\'","ὰ̆":"a\\'","ὰ̄̆":"a\\'","ά":"a/","ά̄":"a/&","ά̆":"a/'","ά̄̆":"a/&","Ἀ":"*)a","Ἀ̄":"*ā","Ἀ̆":"*ă","Ἀ̄̆":"*ā̆","α":"a","ᾱ":"a&","ᾰ":"a'","ᾱ̆":"a&'","ᾶ":"a=","ᾶ̄":"ā","ᾶ̆":"ă","ᾶ̄̆":"ā̆","ἀ":"a)","ἀ̄":"a)&","ἀ̆":"a)'","ἀ̄̆":"a)&","ἁ":"a(","ἁ̄":"a(&","ἁ̆":"a('","ἁ̄̆":"a(&","Ἄ":"*)/a","Ἄ̄":"*ā","Ἄ̆":"*ă","Ἄ̄̆":"*ā̆","ἂ":"a)\\","ἂ̄":"a)\\'","ἂ̆":"a)\\'","ἂ̄̆":"a)\\'","ᾷ":"a=|","ᾷ̄":"ā","ᾷ̆":"ă","ᾷ̄̆":"ā̆","Α":"*a","Ᾱ":"*a&","Ᾰ":"*a'","Ᾱ̆":"*a&'","ἅ":"a(/","ἅ̄":"a(/&","ἅ̆":"a(/'","ἅ̄̆":"a(/&","ἆ":"a)=","ἆ̄":"ā","ἆ̆":"ă","ἆ̄̆":"ā̆","ᾳ":"a|","ᾱͅ":"a|&","ᾰͅ":"a|'","ᾱ̆ͅ":"a|&'","Ἅ":"*(/a","Ἅ̄":"*ā","Ἅ̆":"*ă","Ἅ̄̆":"*ā̆","Ἁ":"*(a","Ἁ̄":"*ā","Ἁ̆":"*ă","Ἁ̄̆":"*ā̆","Ἆ":"*)=a","Ἆ̄":"*ā","Ἆ̆":"*ă","Ἆ̄̆":"*ā̆","ἃ":"a(\\","ἃ̄":"a(\\'","ἃ̆":"a(\\'","ἃ̄̆":"a(\\'","Ἂ":"*)\\a","Ἂ̄":"*ā","Ἂ̆":"*ă","Ἂ̄̆":"*ā̆","ᾇ":"a(=|","ᾇ̄":"ā","ᾇ̆":"ă","ᾇ̄̆":"ā̆","β":"b","Β":"*b","γ":"g","Γ":"*g","δ":"d","Δ":"*d","ε":"e","έ":"e/","ἔ":"e)/","ὲ":"e\\","ἑ":"e(","ἐ":"e)","ἕ":"e(/","Ἕ":"*(/e","Ἐ":"*)e","Ε":"*e","Ἑ":"*(e","Ἔ":"*)/e","ἓ":"e(\\","ἒ":"e)\\","ζ":"z","Ζ":"*z","ῆ":"h=","η":"h","ἣ":"h(\\","ἡ":"h(","ή":"h/","ὴ":"h\\","ἠ":"h)","ἦ":"h)=","ῃ":"h|","ἥ":"h(/","ἢ":"h)\\","ἤ":"h)/","ῇ":"h=|","Ἥ":"*(/h","ᾔ":"h)/|","ἧ":"h(=","ᾗ":"h(=|","Ἠ":"*)h","ῄ":"h/|","ᾕ":"h(/|","Ἡ":"*(h","Ἤ":"*)/h","ῂ":"h\\|","ᾐ":"h)|","ᾖ":"h)=|","ᾑ":"h(|","Η":"*h","Ἦ":"*)=h","Ἣ":"*(\\h","Ἧ":"*(=h","θ":"q","Θ":"*q","ι":"i","ῑ":"i&","ῐ":"i'","ῑ̆":"i&'","ϊ":"i+","ϊ̄":"i+&","ϊ̆":"ĭ","ϊ̄̆":"ī̆&","ί":"i/","ί̄":"i/&","ί̆":"i/'","ί̄̆":"i/&","ῖ":"i=","ῖ̄":"ī","ῖ̆":"ĭ","ῖ̄̆":"ī̆","ἰ":"i)","ἰ̄":"i)&","ἰ̆":"i)'","ἰ̄̆":"i)&","ΐ":"i/+","ΐ̄":"i/+'","ΐ̆":"i/+'","ΐ̄̆":"i/+'","ὶ":"i\\","ὶ̄":"i\\'","ὶ̆":"i\\'","ὶ̄̆":"i\\'","ἱ":"i(","ἱ̄":"i(&","ἱ̆":"i('","ἱ̄̆":"i(&","ἴ":"i)/","ἴ̄":"i)/&","ἴ̆":"i)/'","ἴ̄̆":"i)/&","ἶ":"i)=","ἶ̄":"ī","ἶ̆":"ĭ","ἶ̄̆":"ī̆","ἳ":"i(\\","ἳ̄":"i(\\'","ἳ̆":"i(\\'","ἳ̄̆":"i(\\'","Ἴ":"*)/i","Ἴ̄":"*ī","Ἴ̆":"*ĭ","Ἴ̄̆":"*ī̆","ῒ":"i\\+","ῒ̄":"i\\+'","ῒ̆":"i\\+'","ῒ̄̆":"i\\+'","ἵ":"i(/","ἵ̄":"i(/&","ἵ̆":"i(/'","ἵ̄̆":"i(/&","ἷ":"i(=","ἷ̄":"ī","ἷ̆":"ĭ","ἷ̄̆":"ī̆","Ἰ":"*)i","Ἰ̄":"*ī","Ἰ̆":"*ĭ","Ἰ̄̆":"*ī̆","ἲ":"i)\\","ἲ̄":"i)\\'","ἲ̆":"i)\\'","ἲ̄̆":"i)\\'","Ἱ":"*(i","Ἱ̄":"*ī","Ἱ̆":"*ĭ","Ἱ̄̆":"*ī̆","Ἶ":"*)=i","Ἶ̄":"*ī","Ἶ̆":"*ĭ","Ἶ̄̆":"*ī̆","ῗ":"i=+","ῗ̄":"ī","ῗ̆":"ĭ","ῗ̄̆":"ī̆","Ι":"*i","Ῑ":"*i&","Ῐ":"*i'","Ῑ̆":"*i&'","Ἵ":"*(/i","Ἵ̄":"*ī","Ἵ̆":"*ĭ","Ἵ̄̆":"*ī̆","κ":"k","Κ":"*k","λ":"l","Λ":"*l","μ":"m","Μ":"*m","ν":"n","Ν":"*n","ξ":"c","Ξ":"*c","ο":"o","ὸ":"o\\","ό":"o/","ὃ":"o(\\","ὄ":"o)/","ὀ":"o)","Ὀ":"*)o","Ο":"*o","ὅ":"o(/","ὁ":"o(","Ὄ":"*)/o","Ὅ":"*(/o","Ὃ":"*(\\o","Ὁ":"*(o","Π":"*p","π":"p","ρ":"r","ῥ":"r(","Ῥ":"*(r","ῤ":"r)","Ρ":"*r","ς":"j","σ":"s","Σ":"*s","τ":"t","Τ":"*t","ὐ":"u)","ὐ̄":"u)&","ὐ̆":"u)'","ὐ̄̆":"u)&","υ":"u","ῡ":"u&","ῠ":"u'","ῡ̆":"u&'","ὺ":"u\\","ὺ̄":"u\\'","ὺ̆":"u\\'","ὺ̄̆":"u\\'","ῦ":"u=","ῦ̄":"ū","ῦ̆":"ŭ","ῦ̄̆":"ū̆","ύ":"u/","ύ̄":"u/&","ύ̆":"u/'","ύ̄̆":"u/&","ὗ":"u(=","ὗ̄":"ū","ὗ̆":"ŭ","ὗ̄̆":"ū̆","ὕ":"u(/","ὕ̄":"u(/&","ὕ̆":"u(/'","ὕ̄̆":"u(/&","ϋ":"u+","ϋ̄":"ū","ϋ̆":"ŭ","ϋ̄̆":"ū̆","ὑ":"u(","ὑ̄":"u(&","ὑ̆":"u('","ὑ̄̆":"u(&","ὖ":"u)=","ὖ̄":"ū","ὖ̆":"ŭ","ὖ̄̆":"ū̆","ΰ":"u/+","ΰ̄":"ū","ΰ̆":"ŭ","ΰ̄̆":"ū̆","ὔ":"u)/","ὔ̄":"u)/&","ὔ̆":"u)/'","ὔ̄̆":"u)/&","ῢ":"u\\+","ῢ̄":"ū","ῢ̆":"ŭ","ῢ̄̆":"ū̆","ὓ":"u(\\","ὓ̄":"u(\\'","ὓ̆":"u(\\'","ὓ̄̆":"u(\\'","Ὑ":"*(u","Ὑ̄":"*ū","Ὑ̆":"*ŭ","Ὑ̄̆":"*ū̆","Ὕ":"*(/u","Ὕ̄":"*ū","Ὕ̆":"*ŭ","Ὕ̄̆":"*ū̆","Υ":"*u","Ῡ":"*u&","Ῠ":"*u'","Ῡ̆":"*u&'","φ":"f","Φ":"*f","χ":"x","Χ":"*x","ψ":"y","Ψ":"*y","ω":"w","ώ":"w/","ῶ":"w=","ῳ":"w|","ῷ":"w=|","ὼ":"w\\","ὥ":"w(/","ὣ":"w(\\","ὤ":"w)/","ῴ":"w/|","ᾤ":"w)/|","ὠ":"w)","ὦ":"w)=","ᾧ":"w(=|","ὡ":"w(","ὧ":"w(=","ᾠ":"w)|","Ὠ":"*)w","ὢ":"w)\\","Ὤ":"*)/w","Ὦ":"*)=w","Ὧ":"*(=w","ᾦ":"w)=|","Ω":"*w","Ὥ":"*(/w","Ὣ":"*(\\w","Ὡ":"*(w","Ϝ":"*v","ϝ":"v"},{"a)/":"ἄ","a)/&":"ἄ̄̆","a)/'":"ἄ̆","a\\":"ὰ","a\\'":"ὰ̄̆","a/":"ά","a/&":"ά̄̆","a/'":"ά̆","*)a":"Ἀ","*ā":"Ἂ̄","*ă":"Ἂ̆","*ā̆":"Ἂ̄̆","a":"α","a&":"ᾱ","a'":"ᾰ","a&'":"ᾱ̆","a=":"ᾶ","ā":"ᾇ̄","ă":"ᾇ̆","ā̆":"ᾇ̄̆","a)":"ἀ","a)&":"ἀ̄̆","a)'":"ἀ̆","a(":"ἁ","a(&":"ἁ̄̆","a('":"ἁ̆","*)/a":"Ἄ","a)\\":"ἂ","a)\\'":"ἂ̄̆","a=|":"ᾷ","*a":"Α","*a&":"Ᾱ","*a'":"Ᾰ","*a&'":"Ᾱ̆","a(/":"ἅ","a(/&":"ἅ̄̆","a(/'":"ἅ̆","a)=":"ἆ","a|":"ᾳ","a|&":"ᾱͅ","a|'":"ᾰͅ","a|&'":"ᾱ̆ͅ","*(/a":"Ἅ","*(a":"Ἁ","*)=a":"Ἆ","a(\\":"ἃ","a(\\'":"ἃ̄̆","*)\\a":"Ἂ","a(=|":"ᾇ","b":"β","*b":"Β","g":"γ","*g":"Γ","d":"δ","*d":"Δ","e":"ε","e/":"έ","e)/":"ἔ","e\\":"ὲ","e(":"ἑ","e)":"ἐ","e(/":"ἕ","*(/e":"Ἕ","*)e":"Ἐ","*e":"Ε","*(e":"Ἑ","*)/e":"Ἔ","e(\\":"ἓ","e)\\":"ἒ","z":"ζ","*z":"Ζ","h=":"ῆ","h":"η","h(\\":"ἣ","h(":"ἡ","h/":"ή","h\\":"ὴ","h)":"ἠ","h)=":"ἦ","h|":"ῃ","h(/":"ἥ","h)\\":"ἢ","h)/":"ἤ","h=|":"ῇ","*(/h":"Ἥ","h)/|":"ᾔ","h(=":"ἧ","h(=|":"ᾗ","*)h":"Ἠ","h/|":"ῄ","h(/|":"ᾕ","*(h":"Ἡ","*)/h":"Ἤ","h\\|":"ῂ","h)|":"ᾐ","h)=|":"ᾖ","h(|":"ᾑ","*h":"Η","*)=h":"Ἦ","*(\\h":"Ἣ","*(=h":"Ἧ","q":"θ","*q":"Θ","i":"ι","i&":"ῑ","i'":"ῐ","i&'":"ῑ̆","i+":"ϊ","i+&":"ϊ̄","ĭ":"ῗ̆","ī̆&":"ϊ̄̆","i/":"ί","i/&":"ί̄̆","i/'":"ί̆","i=":"ῖ","ī":"ῗ̄","ī̆":"ῗ̄̆","i)":"ἰ","i)&":"ἰ̄̆","i)'":"ἰ̆","i/+":"ΐ","i/+'":"ΐ̄̆","i\\":"ὶ","i\\'":"ὶ̄̆","i(":"ἱ","i(&":"ἱ̄̆","i('":"ἱ̆","i)/":"ἴ","i)/&":"ἴ̄̆","i)/'":"ἴ̆","i)=":"ἶ","i(\\":"ἳ","i(\\'":"ἳ̄̆","*)/i":"Ἴ","*ī":"Ἵ̄","*ĭ":"Ἵ̆","*ī̆":"Ἵ̄̆","i\\+":"ῒ","i\\+'":"ῒ̄̆","i(/":"ἵ","i(/&":"ἵ̄̆","i(/'":"ἵ̆","i(=":"ἷ","*)i":"Ἰ","i)\\":"ἲ","i)\\'":"ἲ̄̆","*(i":"Ἱ","*)=i":"Ἶ","i=+":"ῗ","*i":"Ι","*i&":"Ῑ","*i'":"Ῐ","*i&'":"Ῑ̆","*(/i":"Ἵ","k":"κ","*k":"Κ","l":"λ","*l":"Λ","m":"μ","*m":"Μ","n":"ν","*n":"Ν","c":"ξ","*c":"Ξ","o":"ο","o\\":"ὸ","o/":"ό","o(\\":"ὃ","o)/":"ὄ","o)":"ὀ","*)o":"Ὀ","*o":"Ο","o(/":"ὅ","o(":"ὁ","*)/o":"Ὄ","*(/o":"Ὅ","*(\\o":"Ὃ","*(o":"Ὁ","*p":"Π","p":"π","r":"ρ","r(":"ῥ","*(r":"Ῥ","r)":"ῤ","*r":"Ρ","j":"ς","s":"σ","*s":"Σ","t":"τ","*t":"Τ","u)":"ὐ","u)&":"ὐ̄̆","u)'":"ὐ̆","u":"υ","u&":"ῡ","u'":"ῠ","u&'":"ῡ̆","u\\":"ὺ","u\\'":"ὺ̄̆","u=":"ῦ","ū":"ῢ̄","ŭ":"ῢ̆","ū̆":"ῢ̄̆","u/":"ύ","u/&":"ύ̄̆","u/'":"ύ̆","u(=":"ὗ","u(/":"ὕ","u(/&":"ὕ̄̆","u(/'":"ὕ̆","u+":"ϋ","u(":"ὑ","u(&":"ὑ̄̆","u('":"ὑ̆","u)=":"ὖ","u/+":"ΰ","u)/":"ὔ","u)/&":"ὔ̄̆","u)/'":"ὔ̆","u\\+":"ῢ","u(\\":"ὓ","u(\\'":"ὓ̄̆","*(u":"Ὑ","*ū":"Ὕ̄","*ŭ":"Ὕ̆","*ū̆":"Ὕ̄̆","*(/u":"Ὕ","*u":"Υ","*u&":"Ῡ","*u'":"Ῠ","*u&'":"Ῡ̆","f":"φ","*f":"Φ","x":"χ","*x":"Χ","y":"ψ","*y":"Ψ","w":"ω","w/":"ώ","w=":"ῶ","w|":"ῳ","w=|":"ῷ","w\\":"ὼ","w(/":"ὥ","w(\\":"ὣ","w)/":"ὤ","w/|":"ῴ","w)/|":"ᾤ","w)":"ὠ","w)=":"ὦ","w(=|":"ᾧ","w(":"ὡ","w(=":"ὧ","w)|":"ᾠ","*)w":"Ὠ","w)\\":"ὢ","*)/w":"Ὤ","*)=w":"Ὦ","*(=w":"Ὧ","w)=|":"ᾦ","*w":"Ω","*(/w":"Ὥ","*(\\w":"Ὣ","*(w":"Ὡ","*v":"Ϝ","v":"ϝ"}]
+JSON
+@@beta_code_conversion = nil
+
+  def Tinycus.beta_code_conversion_table
+    if @@beta_code_conversion.nil? then @@beta_code_conversion=JSON.parse(@@beta_code_conversion_json) end
+    return @@beta_code_conversion
+  end
+
+  def Tinycus.greek_char_unicode_to_beta_code(u)
+    b = Tinycus.beta_code_conversion_table()[0][u]
+    if !b.nil? then return b else return u end # most failures will just be whitespace, punctuation, etc.
+  end
+
+  def Tinycus.greek_char_beta_code_to_unicode(b)
+    b = Tinycus.canonicalize_char_greek_beta_code(b)
+    u = Tinycus.beta_code_conversion_table()[1][b]
+    if !u.nil? then return u else return b end
+  end
+
+  def Tinycus.canonicalize_char_greek_beta_code(b)
+    # Breathing normally comes after accent, but sometimes you see things in the wild where it's reversed.
+    # I can't find any documentation for any preferred or canonical order. What circumstantial evidence I could find I put into the WP article.
+    # Note that the order of |+ doesn't matter, because the same letter can't have both.
+    ")(/\\=|+&'".chars.each { |c|
+      if b=~/(.*)#{Regexp::quote(c)}(.*)/ then b = $1+$2+c end
+    }
+    b.sub!(/\*([a-zA-Z]\d?)([\/\\=)(]+)/) { '*'+$2+$1 } # if its capitalized, move tonal accents and breathing before the letter
+    return b
+  end
+
+  def Tinycus.greek_unicode_to_beta_code(u)
+    u = Tinycus::Cleanup.clean_up_grotty_greek(u,allow_latin:true,clean_perseus:true,standardize_punctuation:false)
+    # ... the conversion below will not work on unicode that isn't done cleanly and according to modern standards
+    b = ''
+    u.chars.each { |uc|
+      b += Tinycus.greek_char_unicode_to_beta_code(uc)
+    }
+    b.tr!("·—’",":_'") # not implemeting keraia
+    return b
+  end
+
+  def Tinycus.greek_beta_code_to_unicode(b)
+    # This implementation will be kind of slow because it does regex replacements in place.
+    b = b.clone
+    b.gsub!(/[sS][1-3]/,'s') # final sigma supported, actual lunate sigma (which looks like c) not supported
+    # cons = "ϝβγδζθκλμνξπρσςτφχψh"
+    b.gsub!(/(?<=[bcdfghjklmnpqrstvwxzBCDFGHJKLMNPQRSTVWXZ])'/,"’") # after a consonant, is apostrophe, not breve
+    b.gsub!(/\*?[sS](?![1-3a-zA-Z*])/,'j') # make non-final sigma into final sigma if it's at end of word
+    b.tr!(":_","·—") # not implemeting keraia
+    b.gsub!(/(\*[)(\\\/=]*[a-zA-Z][)(\/=\\+|&'\´]*)/) {  Tinycus.greek_char_beta_code_to_unicode($1) } # uppercase, expect breathing and tonal before vowel
+    b.gsub!(/([a-zA-Z][)(\/=\\+|&'\´]*)/) {  Tinycus.greek_char_beta_code_to_unicode($1) } # lowercase, breathing after vowel
+    b.gsub!(/ς’/,'σ’')
+    return b
+  end
+
+  def Tinycus.disassemble_greek_char(c)
+    # Returns [plain,d], where plain is a lowercase, unaccented Greek letter (α-ω, plus ς), and d is
+    # a hash with the following keys:
+    #   uppercase, diar, iota_subscript - boolean values
+    #   tonal - string value: none acute grave circumflex
+    #   breathing - string value: none smooth rough
+    # Doesn't handle macrons and breves. I have a function IfMows.disassemble_char in Ifthimos that does that.
+    x = Tinycus.disassemble_greek_char_binary(c)
+    if x.nil? then return nil end
+    plain,decor = x
+    d = {}
+    d['uppercase'] = (decor & 0b1)!=0
+    d['diar'] = (decor & 0b10)!=0
+    d['iota_subscript'] = (decor & 0b100)!=0
+    accent = (decor & 0b11000)>>3
+    if accent==0b00 then d['tonal']='none' end
+    if accent==0b01 then d['tonal']='acute' end
+    if accent==0b10 then d['tonal']='grave' end
+    if accent==0b11 then d['tonal']='circumflex' end
+    if !d.has_key?('tonal') then raise "wtf? #{accent}" end
+    breathing = (decor & 0b1100000)>>5
+    if breathing==0b00 then d['breathing']='none' end
+    if breathing==0b01 then d['breathing']='smooth' end
+    if breathing==0b10 then d['breathing']='rough' end
+    return [plain,d]
+  end
+
+  def Tinycus.assemble_greek_char(plain,d)
+    # The inverse of Tinycus.disassemble_greek_char.
+    # Doesn't handle macrons and breves. I have a function IfMows.assemble_char in Ifthimos that does that.
+    b = 0
+    b |=        0b1 if d['uppercase']
+    b |=       0b10 if d['diar']
+    b |=      0b100 if d['iota_subscript']
+    b |=     0b1000 if d['tonal']=='acute'
+    b |=    0b10000 if d['tonal']=='grave'
+    b |=    0b11000 if d['tonal']=='circumflex'
+    b |=  0b0100000 if d['breathing']=='smooth'
+    b |=  0b1000000 if d['breathing']=='rough'
+    x = plain+("%.2x" % b)
+    return Tinycus.assemble_greek_char_hex(x)
+  end
+
+  def Tinycus.disassemble_greek_char_binary(c)
+    # Returns [plain,b], where plain is a lowercase, unaccented Greek letter (α-ω, plus ς), and b is
+    # an integer containing a set of flags encoded in binary, as follows:
+    #  b |=        0b1 if d['uppercase']
+    #  b |=       0b10 if d['diar']
+    #  b |=      0b100 if d['iota_subscript']
+    #  b |=     0b1000 if d['tonal']=='acute'
+    #  b |=    0b10000 if d['tonal']=='grave'
+    #  b |=    0b11000 if d['tonal']=='circumflex'
+    #  b |=  0b0100000 if d['breathing']=='smooth'
+    #  b |=  0b1000000 if d['breathing']=='rough'
+    return @@disassemble_greek_char_hash[c]
+  end
+
+  def Tinycus.assemble_greek_char_hex(x)
+    # The inverse of the map in Tinycus.disassemble_greek_char_binary.
+    # Accepts an input such as 'α08', where the second and third characters are the hex representation of of
+    # the set of flags described in the comments in the forward map.
+    return @@assemble_greek_char_hash[x]
+  end
+
+# The following are output by generating/assemble_disassemble.rb .
+@@disassemble_greek_char_hash = {
+  'α'=>['α',0b0],
+  'Α'=>['α',0b1],
+  'ᾳ'=>['α',0b100],
+  'ά'=>['α',0b1000],
+  'ὰ'=>['α',0b10000],
+  'ᾶ'=>['α',0b11000],
+  'ᾷ'=>['α',0b11100],
+  'ἀ'=>['α',0b100000],
+  'Ἀ'=>['α',0b100001],
+  'ἄ'=>['α',0b101000],
+  'Ἄ'=>['α',0b101001],
+  'ᾄ'=>['α',0b101100],
+  'ἂ'=>['α',0b110000],
+  'Ἂ'=>['α',0b110001],
+  'ἆ'=>['α',0b111000],
+  'Ἆ'=>['α',0b111001],
+  'ἁ'=>['α',0b1000000],
+  'Ἁ'=>['α',0b1000001],
+  'ἅ'=>['α',0b1001000],
+  'Ἅ'=>['α',0b1001001],
+  'ᾅ'=>['α',0b1001100],
+  'ἃ'=>['α',0b1010000],
+  'ᾇ'=>['α',0b1011100],
+  'β'=>['β',0b0],
+  'Β'=>['β',0b1],
+  'γ'=>['γ',0b0],
+  'Γ'=>['γ',0b1],
+  'δ'=>['δ',0b0],
+  'Δ'=>['δ',0b1],
+  'ε'=>['ε',0b0],
+  'Ε'=>['ε',0b1],
+  'έ'=>['ε',0b1000],
+  'ὲ'=>['ε',0b10000],
+  'ἐ'=>['ε',0b100000],
+  'Ἐ'=>['ε',0b100001],
+  'ἔ'=>['ε',0b101000],
+  'Ἔ'=>['ε',0b101001],
+  'ἒ'=>['ε',0b110000],
+  'ἑ'=>['ε',0b1000000],
+  'Ἑ'=>['ε',0b1000001],
+  'ἕ'=>['ε',0b1001000],
+  'Ἕ'=>['ε',0b1001001],
+  'ἓ'=>['ε',0b1010000],
+  'ζ'=>['ζ',0b0],
+  'Ζ'=>['ζ',0b1],
+  'η'=>['η',0b0],
+  'Η'=>['η',0b1],
+  'ῃ'=>['η',0b100],
+  'ή'=>['η',0b1000],
+  'ῄ'=>['η',0b1100],
+  'ὴ'=>['η',0b10000],
+  'ῂ'=>['η',0b10100],
+  'ῆ'=>['η',0b11000],
+  'ῇ'=>['η',0b11100],
+  'ἠ'=>['η',0b100000],
+  'Ἠ'=>['η',0b100001],
+  'ᾐ'=>['η',0b100100],
+  'ἤ'=>['η',0b101000],
+  'Ἤ'=>['η',0b101001],
+  'ᾔ'=>['η',0b101100],
+  'ἢ'=>['η',0b110000],
+  'ἦ'=>['η',0b111000],
+  'Ἦ'=>['η',0b111001],
+  'ᾖ'=>['η',0b111100],
+  'ἡ'=>['η',0b1000000],
+  'Ἡ'=>['η',0b1000001],
+  'ᾑ'=>['η',0b1000100],
+  'ἥ'=>['η',0b1001000],
+  'Ἥ'=>['η',0b1001001],
+  'ᾕ'=>['η',0b1001100],
+  'ἣ'=>['η',0b1010000],
+  'Ἣ'=>['η',0b1010001],
+  'ἧ'=>['η',0b1011000],
+  'Ἧ'=>['η',0b1011001],
+  'ᾗ'=>['η',0b1011100],
+  'θ'=>['θ',0b0],
+  'Θ'=>['θ',0b1],
+  'ι'=>['ι',0b0],
+  'Ι'=>['ι',0b1],
+  'ϊ'=>['ι',0b10],
+  'ί'=>['ι',0b1000],
+  'ΐ'=>['ι',0b1010],
+  'ὶ'=>['ι',0b10000],
+  'ῒ'=>['ι',0b10010],
+  'ῖ'=>['ι',0b11000],
+  'ῗ'=>['ι',0b11010],
+  'ἰ'=>['ι',0b100000],
+  'Ἰ'=>['ι',0b100001],
+  'ἴ'=>['ι',0b101000],
+  'Ἴ'=>['ι',0b101001],
+  'ἲ'=>['ι',0b110000],
+  'ἶ'=>['ι',0b111000],
+  'Ἶ'=>['ι',0b111001],
+  'ἱ'=>['ι',0b1000000],
+  'Ἱ'=>['ι',0b1000001],
+  'ἵ'=>['ι',0b1001000],
+  'Ἵ'=>['ι',0b1001001],
+  'ἳ'=>['ι',0b1010000],
+  'ἷ'=>['ι',0b1011000],
+  'κ'=>['κ',0b0],
+  'Κ'=>['κ',0b1],
+  'λ'=>['λ',0b0],
+  'Λ'=>['λ',0b1],
+  'μ'=>['μ',0b0],
+  'Μ'=>['μ',0b1],
+  'ν'=>['ν',0b0],
+  'Ν'=>['ν',0b1],
+  'ξ'=>['ξ',0b0],
+  'Ξ'=>['ξ',0b1],
+  'ο'=>['ο',0b0],
+  'Ο'=>['ο',0b1],
+  'ό'=>['ο',0b1000],
+  'ὸ'=>['ο',0b10000],
+  'ὀ'=>['ο',0b100000],
+  'Ὀ'=>['ο',0b100001],
+  'ὄ'=>['ο',0b101000],
+  'Ὄ'=>['ο',0b101001],
+  'ὁ'=>['ο',0b1000000],
+  'Ὁ'=>['ο',0b1000001],
+  'ὅ'=>['ο',0b1001000],
+  'Ὅ'=>['ο',0b1001001],
+  'ὃ'=>['ο',0b1010000],
+  'Ὃ'=>['ο',0b1010001],
+  'π'=>['π',0b0],
+  'Π'=>['π',0b1],
+  'ρ'=>['ρ',0b0],
+  'Ρ'=>['ρ',0b1],
+  'ῤ'=>['ρ',0b100000],
+  'ῥ'=>['ρ',0b1000000],
+  'Ῥ'=>['ρ',0b1000001],
+  'ς'=>['ς',0b0],
+  'σ'=>['σ',0b0],
+  'Σ'=>['σ',0b1],
+  'τ'=>['τ',0b0],
+  'Τ'=>['τ',0b1],
+  'υ'=>['υ',0b0],
+  'Υ'=>['υ',0b1],
+  'ϋ'=>['υ',0b10],
+  'ύ'=>['υ',0b1000],
+  'ΰ'=>['υ',0b1010],
+  'ὺ'=>['υ',0b10000],
+  'ῢ'=>['υ',0b10010],
+  'ῦ'=>['υ',0b11000],
+  'ὐ'=>['υ',0b100000],
+  'ὔ'=>['υ',0b101000],
+  'ὖ'=>['υ',0b111000],
+  'ὑ'=>['υ',0b1000000],
+  'Ὑ'=>['υ',0b1000001],
+  'ὕ'=>['υ',0b1001000],
+  'Ὕ'=>['υ',0b1001001],
+  'ὓ'=>['υ',0b1010000],
+  'ὗ'=>['υ',0b1011000],
+  'φ'=>['φ',0b0],
+  'Φ'=>['φ',0b1],
+  'χ'=>['χ',0b0],
+  'Χ'=>['χ',0b1],
+  'ψ'=>['ψ',0b0],
+  'Ψ'=>['ψ',0b1],
+  'ω'=>['ω',0b0],
+  'Ω'=>['ω',0b1],
+  'ῳ'=>['ω',0b100],
+  'ώ'=>['ω',0b1000],
+  'ῴ'=>['ω',0b1100],
+  'ὼ'=>['ω',0b10000],
+  'ῶ'=>['ω',0b11000],
+  'ῷ'=>['ω',0b11100],
+  'ὠ'=>['ω',0b100000],
+  'Ὠ'=>['ω',0b100001],
+  'ᾠ'=>['ω',0b100100],
+  'ὤ'=>['ω',0b101000],
+  'Ὤ'=>['ω',0b101001],
+  'ᾤ'=>['ω',0b101100],
+  'ὢ'=>['ω',0b110000],
+  'ὦ'=>['ω',0b111000],
+  'Ὦ'=>['ω',0b111001],
+  'ᾦ'=>['ω',0b111100],
+  'ὡ'=>['ω',0b1000000],
+  'Ὡ'=>['ω',0b1000001],
+  'ὥ'=>['ω',0b1001000],
+  'Ὥ'=>['ω',0b1001001],
+  'ὣ'=>['ω',0b1010000],
+  'Ὣ'=>['ω',0b1010001],
+  'ὧ'=>['ω',0b1011000],
+  'Ὧ'=>['ω',0b1011001],
+  'ᾧ'=>['ω',0b1011100],
+  'ϝ'=>['ϝ',0b0],
+  'Ϝ'=>['ϝ',0b1]
+}
+@@assemble_greek_char_hash = {
+
+  'α00'=>'α',
+  'α01'=>'Α',
+  'α04'=>'ᾳ',
+  'α08'=>'ά',
+  'α10'=>'ὰ',
+  'α18'=>'ᾶ',
+  'α1c'=>'ᾷ',
+  'α20'=>'ἀ',
+  'α21'=>'Ἀ',
+  'α28'=>'ἄ',
+  'α29'=>'Ἄ',
+  'α2c'=>'ᾄ',
+  'α30'=>'ἂ',
+  'α31'=>'Ἂ',
+  'α38'=>'ἆ',
+  'α39'=>'Ἆ',
+  'α40'=>'ἁ',
+  'α41'=>'Ἁ',
+  'α48'=>'ἅ',
+  'α49'=>'Ἅ',
+  'α4c'=>'ᾅ',
+  'α50'=>'ἃ',
+  'α5c'=>'ᾇ',
+  'β00'=>'β',
+  'β01'=>'Β',
+  'γ00'=>'γ',
+  'γ01'=>'Γ',
+  'δ00'=>'δ',
+  'δ01'=>'Δ',
+  'ε00'=>'ε',
+  'ε01'=>'Ε',
+  'ε08'=>'έ',
+  'ε10'=>'ὲ',
+  'ε20'=>'ἐ',
+  'ε21'=>'Ἐ',
+  'ε28'=>'ἔ',
+  'ε29'=>'Ἔ',
+  'ε30'=>'ἒ',
+  'ε40'=>'ἑ',
+  'ε41'=>'Ἑ',
+  'ε48'=>'ἕ',
+  'ε49'=>'Ἕ',
+  'ε50'=>'ἓ',
+  'ζ00'=>'ζ',
+  'ζ01'=>'Ζ',
+  'η00'=>'η',
+  'η01'=>'Η',
+  'η04'=>'ῃ',
+  'η08'=>'ή',
+  'η0c'=>'ῄ',
+  'η10'=>'ὴ',
+  'η14'=>'ῂ',
+  'η18'=>'ῆ',
+  'η1c'=>'ῇ',
+  'η20'=>'ἠ',
+  'η21'=>'Ἠ',
+  'η24'=>'ᾐ',
+  'η28'=>'ἤ',
+  'η29'=>'Ἤ',
+  'η2c'=>'ᾔ',
+  'η30'=>'ἢ',
+  'η38'=>'ἦ',
+  'η39'=>'Ἦ',
+  'η3c'=>'ᾖ',
+  'η40'=>'ἡ',
+  'η41'=>'Ἡ',
+  'η44'=>'ᾑ',
+  'η48'=>'ἥ',
+  'η49'=>'Ἥ',
+  'η4c'=>'ᾕ',
+  'η50'=>'ἣ',
+  'η51'=>'Ἣ',
+  'η58'=>'ἧ',
+  'η59'=>'Ἧ',
+  'η5c'=>'ᾗ',
+  'θ00'=>'θ',
+  'θ01'=>'Θ',
+  'ι00'=>'ι',
+  'ι01'=>'Ι',
+  'ι02'=>'ϊ',
+  'ι08'=>'ί',
+  'ι0a'=>'ΐ',
+  'ι10'=>'ὶ',
+  'ι12'=>'ῒ',
+  'ι18'=>'ῖ',
+  'ι1a'=>'ῗ',
+  'ι20'=>'ἰ',
+  'ι21'=>'Ἰ',
+  'ι28'=>'ἴ',
+  'ι29'=>'Ἴ',
+  'ι30'=>'ἲ',
+  'ι38'=>'ἶ',
+  'ι39'=>'Ἶ',
+  'ι40'=>'ἱ',
+  'ι41'=>'Ἱ',
+  'ι48'=>'ἵ',
+  'ι49'=>'Ἵ',
+  'ι50'=>'ἳ',
+  'ι58'=>'ἷ',
+  'κ00'=>'κ',
+  'κ01'=>'Κ',
+  'λ00'=>'λ',
+  'λ01'=>'Λ',
+  'μ00'=>'μ',
+  'μ01'=>'Μ',
+  'ν00'=>'ν',
+  'ν01'=>'Ν',
+  'ξ00'=>'ξ',
+  'ξ01'=>'Ξ',
+  'ο00'=>'ο',
+  'ο01'=>'Ο',
+  'ο08'=>'ό',
+  'ο10'=>'ὸ',
+  'ο20'=>'ὀ',
+  'ο21'=>'Ὀ',
+  'ο28'=>'ὄ',
+  'ο29'=>'Ὄ',
+  'ο40'=>'ὁ',
+  'ο41'=>'Ὁ',
+  'ο48'=>'ὅ',
+  'ο49'=>'Ὅ',
+  'ο50'=>'ὃ',
+  'ο51'=>'Ὃ',
+  'π00'=>'π',
+  'π01'=>'Π',
+  'ρ00'=>'ρ',
+  'ρ01'=>'Ρ',
+  'ρ20'=>'ῤ',
+  'ρ40'=>'ῥ',
+  'ρ41'=>'Ῥ',
+  'ς00'=>'ς',
+  'σ00'=>'σ',
+  'σ01'=>'Σ',
+  'τ00'=>'τ',
+  'τ01'=>'Τ',
+  'υ00'=>'υ',
+  'υ01'=>'Υ',
+  'υ02'=>'ϋ',
+  'υ08'=>'ύ',
+  'υ0a'=>'ΰ',
+  'υ10'=>'ὺ',
+  'υ12'=>'ῢ',
+  'υ18'=>'ῦ',
+  'υ20'=>'ὐ',
+  'υ28'=>'ὔ',
+  'υ38'=>'ὖ',
+  'υ40'=>'ὑ',
+  'υ41'=>'Ὑ',
+  'υ48'=>'ὕ',
+  'υ49'=>'Ὕ',
+  'υ50'=>'ὓ',
+  'υ58'=>'ὗ',
+  'φ00'=>'φ',
+  'φ01'=>'Φ',
+  'χ00'=>'χ',
+  'χ01'=>'Χ',
+  'ψ00'=>'ψ',
+  'ψ01'=>'Ψ',
+  'ω00'=>'ω',
+  'ω01'=>'Ω',
+  'ω04'=>'ῳ',
+  'ω08'=>'ώ',
+  'ω0c'=>'ῴ',
+  'ω10'=>'ὼ',
+  'ω18'=>'ῶ',
+  'ω1c'=>'ῷ',
+  'ω20'=>'ὠ',
+  'ω21'=>'Ὠ',
+  'ω24'=>'ᾠ',
+  'ω28'=>'ὤ',
+  'ω29'=>'Ὤ',
+  'ω2c'=>'ᾤ',
+  'ω30'=>'ὢ',
+  'ω38'=>'ὦ',
+  'ω39'=>'Ὦ',
+  'ω3c'=>'ᾦ',
+  'ω40'=>'ὡ',
+  'ω41'=>'Ὡ',
+  'ω48'=>'ὥ',
+  'ω49'=>'Ὥ',
+  'ω50'=>'ὣ',
+  'ω51'=>'Ὣ',
+  'ω58'=>'ὧ',
+  'ω59'=>'Ὧ',
+  'ω5c'=>'ᾧ',
+  'ϝ00'=>'ϝ',
+  'ϝ01'=>'Ϝ'
+}
+
+end # module Tinycus
+