util 0.2.0

data/test/unit.rb

@@ -0,0 +1,20 @@
+require 'util/test'
+
+$test = Util::Testing.new
+
+paths = [File.join(__dir__, 'unit')]
+files = []
+begin
+  path = paths.shift
+  Dir[File.join path, '*'].each do |f|
+    next if f == '.' or f == '..'
+    paths << f if File.directory? f
+    files << f if File.file? f and File.extname(f) == '.rb'
+  end
+end until paths.empty?
+
+files.each do |f|
+  require_relative f.sub(__dir__, '')[1..-1]
+end
+
+$test.run

data/test/unit/i18n.rb

@@ -0,0 +1,216 @@
+$test.register 'i18n-init', [false, true] do ||
+  require 'util/i18n'
+  result = []
+  result << Util::I18n.initialized?
+  Util::I18n.init
+  result << Util::I18n.initialized?
+end
+
+results = [true, :eng, true, :fra, true, :prv, true, :eng]
+results += [true, :zho, true, :oci, true, :sqi, true, :qqa]
+results += [false, :qqa, false, :qqa]
+$test.register 'i18n-default-lang', results do ||
+  require 'util/i18n'
+  [ Util::I18n.set_default_lang(:eng),
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang('fra'), # String
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:PrV), # Non lowercase
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang('en_US.UTF-8'), # POSIX
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang('zh-Hant-344'), # IETF
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:oc), # ISO 639-1
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:alb), # ISO 639-2
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:qqa), # Private use
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:aaj), # Non existent code
+    Util::I18n.default_lang,
+    Util::I18n.set_default_lang(:afa), # ISO 639-2 code with
+    Util::I18n.default_lang,           # no ISO 639-3 equivalent
+  ]
+end
+
+results = [[:set_def_unknown, :aaj], [:set_def_unknown, :afa], nil]
+$test.register 'i18n-next-error', results do ||
+  require 'util/i18n'
+  Util::I18n.init # Purge all previous errors
+  Util::I18n.set_default_lang(:aaj)
+  Util::I18n.set_default_lang(:afa)
+  [Util::I18n.next_error, Util::I18n.next_error, Util::I18n.next_error]
+end
+
+results = ['alias', '今日は', 'la-vie-de-ma-mère', 'ça-va-oui-et-toi']
+results += ['Pz8hPw==', 'hello-darkneß-m41_0-d']
+results += ['estie-de-tabarnak-ISHCpyE=', '']
+$test.register 'i18n-slugify', results do ||
+  require 'util/i18n'
+  [ Util::I18n.send(:slugify, 'alias'),
+    Util::I18n.send(:slugify, '今日は'),
+    Util::I18n.send(:slugify, 'La vie de ma mère'),
+    Util::I18n.send(:slugify, 'Ça va? Oui, et toi?'),
+    Util::I18n.send(:slugify, '??!?'),
+    Util::I18n.send(:slugify, File.join('hello', 'Darkneß', 'm41_0!d')),
+    Util::I18n.send(:slugify, File.join('Estie', 'de', 'tabarnak', '!!§!')),
+    Util::I18n.send(:slugify, ''),
+  ]
+end
+
+results = [false, :reg_no_name, false, :reg_path_not_exist]
+results += [false, :reg_path_not_exist, false, :reg_path_not_dir]
+results += [false, :reg_path_not_dir, 'temp', (File::SEPARATOR + 'i18n')]
+results += [true, false, true]
+$test.register 'i18n-register-check-args', results do ||
+  require 'util/i18n'
+  Util::I18n.init # Purge all previous errors
+  res = [Util::I18n.register, Util::I18n.next_error[0]]
+    # No name given
+  base = File.dirname(File.expand_path __FILE__)
+  path = File.join base, 'i18n', 'CamelCase', 'temp.rb'
+  res += [Util::I18n.register('temp', path), Util::I18n.next_error[0]]
+    # Will search inside 'i18n/CamelCase/i18n', that does not exist
+  path = File.join base, 'i18n', 'CamelCase', 'i18n'
+  res += [Util::I18n.register('temp', path, false), Util::I18n.next_error[0]]
+    # Same with an absolute path
+  path = File.join base, 'i18n', 'temp.rb'
+  res += [Util::I18n.register('temp', path), Util::I18n.next_error[0]]
+    # Will search inside 'i18n/i18n', that is a file not a folder
+  path = File.join base, 'i18n', 'i18n'
+  res += [Util::I18n.register('temp', path, false), Util::I18n.next_error[0]]
+    # Same with an absolute path
+  temp = Util::I18n.send(:register_check_args, 'temp', __FILE__, true, false)
+  res += [temp[0], temp[1].sub(base, ''), temp[2], temp[3]]
+    # The normal most ordinary case
+  temp = Util::I18n.send(:register_check_args, 'temp', '', true, false)
+  if temp == false then
+    err_name, err_data = Util::I18n.next_error
+    right_err = (err_name == :reg_path_not_exist or :reg_path_not_dir)
+    err_data = err_data.sub(File.expand_path('.'), '')
+    right_path = err_data == (File::SEPARATOR + 'i18n')
+    res << (right_err and right_path)
+  else
+    path = temp[1].sub(File.expand_path('.'), '')
+    res << (temp[0] == 'temp' and path == (File::SEPARATOR + 'i18n') \
+      and temp[2] == true and temp[3] == false)
+  end # Much more difficult to check: when `path` is empty
+  res
+end
+
+path = File.join File.dirname(File.expand_path __FILE__), 'i18n'
+results = [File.join(path, '*.yml'), 'temp']
+results += [File.join(path, '*.yml'), 'temp']
+results += [File.join(path, 'αλιας', '*.yml'), 'temp-αλιας']
+results += [File.join(path, 'void', '*.yml'), 'temp-void']
+results += [File.join(path, 'CamelCase', '*.yml'), 'temp-camelcase']
+results += [File.join(path, 'void', 'fra.yml', '*.yml'), 'temp-void-fra-yml']
+path = nil
+$test.register 'i18n-register-get-locations', results do ||
+  require 'util/i18n'
+  path = File.join File.dirname(File.expand_path __FILE__), 'i18n'
+  [ Util::I18n.send(:register_get_locations, 'temp', path, false),
+    Util::I18n.send(:register_get_locations, 'temp', path, true)
+  ].flatten
+end
+
+hash = { 'temp' => {} }
+hash['temp'][:eng] = { 
+  'toto' => 'tata mais en anglais',
+  'titi'=> 'tutu mais en anglais'
+}
+hash['temp'][:fra] = { 'toto' => 'tata', 'titi' => 'tutu' }
+hash['temp'][:prv] = { 'toto' => 'pecaire', 'titi' => 'eh fada' }
+results = [Marshal.load(Marshal.dump(hash))] # Deep copy
+hash['temp'][:fra]['tété'] = 'toto'
+results << Marshal.load(Marshal.dump(hash))
+results += [{}, :reg_yaml_cant_open]
+hash = nil
+$test.register 'i18n-register-get-messages', results do ||
+  require 'util/i18n'
+  Util::I18n.init # Purge all previous errors
+  path = File.join File.dirname(File.expand_path __FILE__), 'i18n'
+  loc = [File.join(path, '*.yml'), 'temp']
+  messages = {}
+  res = [Util::I18n.send(:register_get_messages, loc, messages)]
+    # Normal registering
+  messages = { 'temp' => { :fra => { 'toto' => 'titi', 'tété' => 'toto' } } }
+  res << Util::I18n.send(:register_get_messages, loc, messages)
+    # When internationalization tokens have already been registered
+  loc = [File.join(path, 'void', '*.yml'), 'temp']
+  messages = {}
+  res << Util::I18n.send(:register_get_messages, loc, messages)
+  res << Util::I18n.next_error[0]
+    # 'i18n/void/fra.yml' is a directory, not a file
+  res
+end
+
+results = [false, :reg_yaml_cant_open, :reg_no_file]
+hash = { 'temp' => {} }
+hash['temp'][:eng] = { 
+  'toto' => 'tata mais en anglais',
+  'titi'=> 'tutu mais en anglais'
+}
+hash['temp'][:fra] = { 'toto' => 'tata', 'titi' => 'tutu' }
+hash['temp'][:prv] = { 'toto' => 'pecaire', 'titi' => 'eh fada' }
+results += [true, Marshal.load(Marshal.dump(hash))] # Deep copy
+hash['temp-αλιας'] = { :fra => hash['temp'][:fra] }
+hash['temp-camelcase'] = {
+  :eng => hash['temp'][:eng],
+  :fra => { 'toto' => 'tata-bis', 'titi' => 'tutu' }
+}
+results += [true, Marshal.load(Marshal.dump(hash))]
+hash = nil
+$test.register 'i18n-register', results do ||
+  require 'util/i18n'
+  Util::I18n.init # Purge all previous errors
+  base = File.join File.dirname(File.expand_path __FILE__), 'i18n'
+  path = File.join base, 'void'
+  res = [Util::I18n.register('temp', path, false)]
+  res += [Util::I18n.next_error[0], Util::I18n.next_error[0]]
+    # No valid file inside 'i18n/void', because 'fra.yml' is a directory
+  res << Util::I18n.register('temp', __FILE__)
+  res << Util::I18n.instance_variable_get('@messages')
+    # Normal registering
+  Util::I18n.init # Reset the @messages variable
+  res << Util::I18n.register('temp', __FILE__, true, true)
+  res << Util::I18n.instance_variable_get('@messages')
+    # Registering recursively
+  res
+end
+
+results = ['', :messages_empty, true, :reg_yaml_cant_open]
+results += ['', :msg_no_module_content, '', :msg_no_valid_lang]
+results += [true, 'tata', 'pecaire', 'tata-bis', 'tata-bis']
+results += ['tata-bis', '']
+$test.register 'i18n-message', results do ||
+  require 'util/i18n'
+  Util::I18n.init # Reset the data and errors
+  Util::I18n.set_default_lang :sqi
+  ov, $VERBOSE = $VERBOSE, nil
+  Util::I18n.const_set :DEFAULT_LANG, :sqi
+  $VERBOSE = ov
+  [ Util::I18n.message('toto'), # Nothing_registered yet
+    Util::I18n.next_error[0],
+    Util::I18n.register('temp', __FILE__, true, true),
+    Util::I18n.next_error[0],
+    Util::I18n.message('toto', mod: 'camelcase'), # No such module
+    Util::I18n.next_error[0],
+    Util::I18n.message('toto', lang: :zho), # No such language, and the
+    Util::I18n.next_error[0], # default languages neither have messages
+    Util::I18n.set_default_lang(:fra),
+    Util::I18n.message('toto'), # Default settings
+    Util::I18n.message('toto', lang: :prv), # Different language
+    Util::I18n.message('toto', mod: 'temp-camelcase'), # Different module
+    Util::I18n.message('toto'), # Last used module has changed
+    Util::I18n.message('toto', lang: :zho), # Message not translated
+    Util::I18n.message('tata'), # Message does not exist
+  ]
+end
+
+$test.register '', [] do ||
+  require 'util/lists/iso639'
+  Util::Lists::ISO639::P3.instance_variable_set '@complete', nil
+  # Put the ISO639 module back to its uninitialized state.
+end

data/test/unit/i18n/CamelCase/eng.yml

@@ -0,0 +1,2 @@
+toto: tata mais en anglais
+titi: tutu mais en anglais

data/test/unit/i18n/CamelCase/fra.yml

@@ -0,0 +1,2 @@
+toto: tata-bis
+titi: tutu

data/test/unit/i18n/aaj.yml

@@ -0,0 +1,2 @@
+toto: تاتا
+titi: توتو

data/test/unit/i18n/en.yml

@@ -0,0 +1,2 @@
+toto: tata mais en anglais
+titi: tutu mais en anglais

data/test/unit/i18n/fra.yml

@@ -0,0 +1,2 @@
+toto: tata
+titi: tutu

data/test/unit/i18n/prv.yml

@@ -0,0 +1,2 @@
+toto: pecaire
+titi: eh fada

data/test/unit/i18n//316/261/316/273/316/271/316/261/317/202/fra.yml

@@ -0,0 +1,2 @@
+toto: tata
+titi: tutu

data/test/unit/lists/iso639.rb

@@ -0,0 +1,78 @@
+$test.register 'iso639-init', [false, true] do ||
+  require 'util/lists/iso639'
+  result = []
+  result << Util::Lists::ISO639::P3.initialized?
+  Util::Lists::ISO639::P3.init
+  result << Util::Lists::ISO639::P3.initialized?
+end
+
+result = [true, true, true, true, false]
+$test.register 'iso639-3-exist', result do ||
+  require 'util/lists/iso639'
+  [ Util::Lists::ISO639::P3.exist?(:fra),
+    Util::Lists::ISO639::P3.exist?('fra'),
+    Util::Lists::ISO639::P3.exist?(:prv),
+    Util::Lists::ISO639::P3.exist?(:qst),
+    Util::Lists::ISO639::P3.exist?(:aaj),
+  ]
+end
+
+result = [false, false, true, false, false]
+$test.register 'iso639-3-deprecated', result do ||
+  require 'util/lists/iso639'
+  [ Util::Lists::ISO639::P3.deprecated?(:fra),
+    Util::Lists::ISO639::P3.deprecated?('fra'),
+    Util::Lists::ISO639::P3.deprecated?(:prv),
+    Util::Lists::ISO639::P3.deprecated?(:qst),
+    Util::Lists::ISO639::P3.deprecated?(:aaj),
+  ]
+end
+
+result = [false, false, false, true, false]
+$test.register 'iso639-3-private', result do ||
+  require 'util/lists/iso639'
+  [ Util::Lists::ISO639::P3.private?(:fra),
+    Util::Lists::ISO639::P3.private?('fra'),
+    Util::Lists::ISO639::P3.private?(:prv),
+    Util::Lists::ISO639::P3.private?(:qst),
+    Util::Lists::ISO639::P3.private?(:aaj),
+  ]
+end
+
+result = [true, true, false, false, false]
+$test.register 'iso639-3-valid', result do ||
+  require 'util/lists/iso639'
+  [ Util::Lists::ISO639::P3.valid?(:fra),
+    Util::Lists::ISO639::P3.valid?('fra'),
+    Util::Lists::ISO639::P3.valid?(:prv),
+    Util::Lists::ISO639::P3.valid?(:qst),
+    Util::Lists::ISO639::P3.valid?(:aaj),
+  ]
+end
+
+result = [:fra, :fra, :eng, nil, nil, nil]
+$test.register 'iso639-3-from1', result do ||
+  require 'util/lists/iso639'
+  class Fr; end
+  [ Util::Lists::ISO639::P3.from1(:fr),
+    Util::Lists::ISO639::P3.from1('fr'),
+    Util::Lists::ISO639::P3.from1(:en),
+    Util::Lists::ISO639::P3.from1(:qs),
+    Util::Lists::ISO639::P3.from1(:pr),
+    Util::Lists::ISO639::P3.from2(Fr),
+  ]
+end
+
+result = [:fra, :fra, :fra, nil, nil, :qst, nil]
+$test.register 'iso639-3-from2', result do ||
+  require 'util/lists/iso639'
+  class Fra; end
+  [ Util::Lists::ISO639::P3.from2(:fra),
+    Util::Lists::ISO639::P3.from2('fra'),
+    Util::Lists::ISO639::P3.from2(:fre),
+    Util::Lists::ISO639::P3.from2(:fr),
+    Util::Lists::ISO639::P3.from2(:prv),
+    Util::Lists::ISO639::P3.from2(:qst),
+    Util::Lists::ISO639::P3.from2(Fra),
+  ]
+end

data/tools/create-iso639-3.rb

@@ -0,0 +1,235 @@
+# Generates a YAML file containing all ISO 639-3 codes and information
+#   about them, from the lists distributed by the SIL.
+#
+# See `$0 --help` for command-line options.
+# See main `LICENSES` file for license information.
+# @author Guillaume Lestringant
+# @version 1.0
+# @note This program is supposed to not raise any exception, so that
+#   it can be called from a script (cf. `-q` option).
+require 'optparse'
+require 'yaml'
+
+def log type, msg
+  code = { :none => "\033[0m", :ok => "\033[32m",
+           :err => "\033[31m", :warn => "\033[33m" }
+  text = { :none => '', :ok => '', :err => '[ERROR] ', :warn => '[WARNING] ' }
+
+  unless $options[:quiet] then
+    $options[:stderr_log] \
+      ? $STDERR.puts("#{code[type]}#{text[type]}#{msg}#{code[:none]}")
+      : puts("#{code[type]}#{text[type]}#{msg}#{code[:none]}")
+  end
+
+  exit 1 if type == :err
+end
+
+    # Parse command-line options
+     ###########################
+
+$options = {
+  :dest => 'iso639-3.yml',
+  :force => false,
+  :quiet => false,
+  :src_b => 'iso639-3',
+  :src_d => 'iso639-3d',
+  :src_m => 'iso639-3m',
+  :stderr_log => false,
+}
+
+OptionParser.new do |opts|
+  DEPR_TEXT = "List of deprecated codes. [DEF: #{$options[:src_d]}(.txt)]"
+  DEST_TEXT = "The file that will be generated. [DEF: #{$options[:dest]}(.txt)]"
+  FORCE_TEXT = 'Write destination even if file already exists. [DEF: false]'
+  HELP_TEXT = 'Display this help message.'
+  MACRO_TEXT = "List of macrolanguage mappings. [DEF: #{$options[:src_m]}(.txt)]"
+  QUIET_TEXT = 'Do not diplay anything in terminal. [DEF: false]'
+  SRC_TEXT = "List of active codes. [DEF: #{$options[:src_b]}(.txt)]"
+  STDERR_TEXT = 'Log to STDERR instead of STDOUT. [DEF: false]'
+  VERSION_TEXT = 'Print program version.'
+  NAME = 'Create ISO 639-3'
+  VERSION = '1.0'
+
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on("-d", "--depr FILE", String, DEPR_TEXT) do |f|
+    $options[:src_d] = f
+  end
+
+  opts.on("-f", "--force", FORCE_TEXT) do |f|
+    $options[:force] = true
+  end
+
+  opts.on("-h", "--help", HELP_TEXT) do |h|
+    puts opts
+    exit
+  end
+
+  opts.on("-l", "--log-to-stderr", STDERR_TEXT) do |f|
+    $options[:stderr_log] = true
+  end
+
+  opts.on("-m", "--macro FILE", String, MACRO_TEXT) do |f|
+    $options[:src_m] = f
+  end
+
+  opts.on("-o", "--output FILE", String, DEST_TEXT) do |f|
+    $options[:dest] = f
+  end
+
+  opts.on("-q", "--quiet", QUIET_TEXT) do |f|
+    $options[:quiet] = true
+  end
+
+  opts.on("-s", "--src FILE", String, SRC_TEXT) do |f|
+    $options[:src_b] = f
+  end
+
+  opts.on("-v", "--version", VERSION_TEXT) do |v|
+    puts "#{NAME} v. #{VERSION}"
+    exit
+  end
+end.parse!
+
+log :none, "#{NAME} v. #{VERSION}"
+
+    # Check all parameters
+     #####################
+
+def check_src idx, default
+  return if File.exist? $options[idx]
+
+  log :warn, "File not found `#{$options[idx]}`."
+  $options[idx] = File.extname($options[idx]) == '.txt' \
+    ? File.basename($options[idx], '.txt')
+    : $options[idx] + '.txt'
+  log :warn, "Trying `#{$options[idx]}` instead."
+  return if File.exist? $options[idx]
+
+  if $options[idx] == default or $options[idx] == (default + '.txt') then
+    log :err, 'File not found. No other possibility. Aborting.'
+  end
+
+  log :warn, "File not found. Reverting to default value `#{default}`."
+  $options[idx] = default
+  return if File.exist? $options[idx]
+
+  $options[idx] += '.txt'
+  log :warn, "File not found. Trying `#{$options[idx]}` instead."
+  return if File.exist? $options[idx]
+
+  log :err, 'File not found. No other possibility. Aborting.'
+end
+
+def mkdir_p dir
+  parent = File.dirname dir
+  log :err, "Root directory not found `#{dir}`." if dir == parent
+  mkdir_p parent unless File.exist? parent
+  log :err, "Not a directory `#{parent}`." unless File.directory? parent
+  Dir.mkdir dir
+end
+
+check_src :src_b, 'iso639-3'
+check_src :src_d, 'iso639-3d'
+check_src :src_m, 'iso639-3m'
+
+if File.exist? $options[:dest] then
+  message = "File already exists `#{$options[:dest]}`. Use `-f` to overwrite."
+  log :err, message unless $options[:force]
+else
+  dir = File.dirname $options[:dest]
+  mkdir_p dir unless File.exist? dir
+  log :err, "Not a directory `#{dir}`." unless File.directory? dir
+end
+
+log :ok, 'All parameters clear.'
+
+    # Extract the data
+     #################
+
+keys = []
+codes = {}
+result = {}
+
+active = File.read($options[:src_b]).lines[1..-1]
+active = [] if active.nil?
+active.each do |l|
+  id, part2b, part2t, part1, scope, type, name, comment = l.chomp.split "\t"
+
+  keys << id
+  id = id.to_sym
+  codes[id] = { :name => name }
+
+  case scope
+  when 'I' then codes[id][:scope] = :individual
+  when 'M' then codes[id][:scope] = :macro
+  when 'S' then codes[id][:scope] = :special
+  else puts "Unknown scope for #{id} : #{scope}."
+  end
+
+  case type
+  when 'A' then codes[id][:type] = :ancient
+  when 'C' then codes[id][:type] = :constructed
+  when 'E' then codes[id][:type] = :extinct
+  when 'H' then codes[id][:type] = :historical
+  when 'L' then codes[id][:type] = :living
+  when 'S' then codes[id][:type] = :special
+  else puts "Unknown type for #{id} : #{type}."
+  end
+
+  codes[id][:p1] = part1.to_sym unless part1.empty?
+  codes[id][:p2b] = part2b.to_sym unless part2b.empty?
+  codes[id][:p2t] = part2t.to_sym unless part2t.empty?
+  codes[id][:comment] = comment unless comment.nil? or comment.empty?
+end
+
+log :ok, 'Extracted active codes.'
+
+deprecated = File.read($options[:src_d]).lines[1..-1]
+deprecated = [] if deprecated.nil?
+deprecated.each do |l|
+  id, name, reason, new, comment, date = l.chomp.split "\t"
+
+  keys << id
+  id = id.to_sym
+  codes[id] = { :deprecated => true, :name => name }
+
+  case reason
+  when 'C' then codes[id][:reason] = :change
+  when 'D' then codes[id][:reason] = :duplicate
+  when 'N' then codes[id][:reason] = :non_existent
+  when 'S' then codes[id][:reason] = :split
+  when 'M' then codes[id][:reason] = :merge
+  else puts "Unknown deprecation reason for #{id} : #{reason}."
+  end
+
+  codes[id][:new] = new.to_sym unless new.empty?
+  codes[id][:comment] = comment unless comment.empty?
+  codes[id][:date] = date unless date.empty? or date.empty?
+end
+
+log :ok, 'Extracted deprecated codes.'
+
+macro = File.read($options[:src_m]).lines[1..-1]
+macro = [] if macro.nil?
+macro.each do |l|
+  macro, member, status = l.chomp.split "\t"
+
+  macro, member = macro.to_sym, member.to_sym
+  codes[macro][:members] = [] if codes[macro][:members].nil?
+  codes[macro][:members] << member
+  codes[member][:macro] = macro
+end
+
+log :ok, 'Extracted macrolanguage mappings.'
+
+    # Export
+     #######
+
+keys.sort.each do |k|
+  result[k.to_sym] = codes[k.to_sym]
+end
+
+File.write $options[:dest], result.to_yaml
+
+log :ok, 'Full success.'