iso-codes 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+pkg
+doc

data/CHANGELOG

@@ -0,0 +1 @@
+v0.1.0. initial release

data/README.rdoc

@@ -0,0 +1,73 @@
+= iso-codes
+
+A comprehensive database of ISO language and script codes.
+
+== Installation
+
+   gem install iso-codes
+
+== Usage
+
+Language codes can be looked up using +find_language+:
+
+   require 'iso_codes'
+
+   language = ISOCodes.find_language('lav')
+   # ISO 639-3 reference name
+   language.reference_name
+   # ISO 639-3 identifier
+   language.identifier
+   # ISO 639-1 (alpha-2) identifier
+   language.alpha2
+   # ISO 639-2 (alpha-3) identifiers
+   language.alpha3_bibliographic
+   language.alpha3_terminology
+
+=== Macrolanguages and individual languages
+
+ISO 639-3 defines language identifiers that covers a set of other language
+identifiers. For example, the identifier +lav+ for Latvian corresponds to
++lvs+ for Standard Latvian and +ltg+ for Latgalian. These identifiers are
+called macrolanguages (see http://www.sil.org/iso639-3/scope.asp#M for a
+more formal definition). See http://www.sil.org/iso639-3/macrolanguages.asp
+for an overview of such identifiers.
+
+Macrolanguage identifiers can be looked up in the same way as other
+language codes:
+
+   language = ISOCodes.find_language('lav')
+   language.reference_name
+   # => "Latvian"
+   language.class
+   # => ISOCodes::Macrolanguage
+   language.alpha3_bibliographic
+   # => "lav"
+   language.alpha2
+   # => "lv"
+
+The individual languages covered by the identifier can be listed:
+
+   >> language.individual_languages.map { |l| l.identifier }
+   # => ["ltg", "lvs"]
+
+For an individual language the macrolanguage, if any, can be found.
+
+   language = ISOCodes.find_language('ltg')
+   language.class
+   # => ISOCodes::IndividualLanguage
+   language.macrolanguage.identifier
+   # => "lav"
+
+== TODO
+
+* ISO 639-1 and ISO 639-2
+* IANA language codes
+* country/region codes (ISO 3166-1 through 3)
+* currencies (ISO 4217:2001 and 2008)
+
+If you feel like implementing any of this, please fork the project on
+github.
+
+= Copyright
+
+Copyright (c) 2010 Marius L. Jøhndal.

data/README.todo

@@ -0,0 +1,39 @@
+== Coverage
+
+The database currently includes codes for the following domains
+
+* languages: two letter (`alpha-2') and three letter (`alpha-3') ISO codes,
+  and human readable names in English and, for a subset, in French.
+
+* scripts: four letter (`alpha-4') and numeric ISO codes, and human readable
+  names in English and French.
+
+=== Standards
+
+* languages:
+
+  - ISO 639-1
+
+  - ISO 639-2:1998 Codes for the representation of names of languages-- Part 2: alpha-3 code.
+
+    Registration authority: http://www.loc.gov/standards/iso639-2/
+    Last update: 2008-07-08 (http://www.loc.gov/standards/iso639-2/php/code_changes.php)
+
+  - ISO 639-3:2007 Codes for the representation of names of languages -- Part 3: Alpha-3 code for comprehensive coverage of languages
+
+    Registration authority: http://www.sil.org/iso639-3/
+    Last update: 2008-07-11
+
+  - RFC 4646 and RFC 4647 (which together obsoleted RFC 3066 and RFC 1766)
+
+    Language subtag registry: http://www.iana.org/assignments/language-subtag-registry
+    Last update: 2008-07-23
+
+* scripts: 
+
+  - ISO 15924:2004 Codes for the representation of names of scripts
+
+    Registration authority: http://unicode.org/iso15924/
+    Last update: 2007-11-26 (http://unicode.org/iso15924/codechanges.html)
+
+== License

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+
+  Jeweler::Tasks.new do |p|
+    p.name = "iso-codes"
+    p.summary = "ISO language and script code support"
+    p.description = "A database of ISO language and script codes."
+    p.authors = ['Marius L. Jøhndal']
+    p.email = "mariuslj (at) ifi [dot] uio (dot) no"
+    p.homepage = "http://github.com/mlj/iso-codes"
+    p.rubyforge_project = "iso-codes"
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/iso-codes.gemspec

@@ -0,0 +1,51 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{iso-codes}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Marius L. J\303\270hndal"]
+  s.date = %q{2010-06-29}
+  s.description = %q{A database of ISO language and script codes.}
+  s.email = %q{mariuslj (at) ifi [dot] uio (dot) no}
+  s.extra_rdoc_files = [
+    "README.rdoc",
+     "README.todo"
+  ]
+  s.files = [
+    ".gitignore",
+     "CHANGELOG",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "iso-codes.gemspec",
+     "lib/iso-639-3-macrolanguages_20100128.tab.gz",
+     "lib/iso-639-3_20100330.tab.gz",
+     "lib/iso_codes.rb",
+     "test/test_iso_codes.rb"
+  ]
+  s.homepage = %q{http://github.com/mlj/iso-codes}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{iso-codes}
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{ISO language and script code support}
+  s.test_files = [
+    "test/test_iso_codes.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/lib/iso_codes.rb

@@ -0,0 +1,178 @@
+#
+# iso_codes.rb - A comprehensive database of ISO codes
+#
+# Written by Marius L. Jøhndal, 2008, 2010.
+#
+require 'zlib'
+
+module ISOCodes
+  # Version of the ISO 639-3 code set (see
+  # http://www.sil.org/iso639-3/download.asp).
+  ISO_639_3_VERSION = '20100330'
+
+  # Version of the ISO 639-3 macrolanguage mappings (see
+  # http://www.sil.org/iso639-3/download.asp).
+  ISO_639_3_MACROLANGUAGE_MAPPINGS_VERSION = '20100128'
+
+  class Language
+    # Returns the ISO 639-3 identifier.
+    attr_reader :identifier
+
+    # Returns the equivalent ISO 639-2 identifier of the bibliographic applications code set, or
+    # +nil+ if none is defined.
+    attr_reader :alpha3_bibliographic
+
+    # Returns the equivalent ISO 639-2 identifier of the terminology applications code set, or
+    # +nil+ if none is defined.
+    attr_reader :alpha3_terminology
+
+    # Returns the equivalent ISO 639-1 identifier, or +nil+ if none is defined.
+    attr_reader :alpha2
+
+    # Returns the language type. Language type is either <tt>:living</tt>,
+    # <tt>:extinct</tt>, <tt>:ancient</tt>, <tt>:historic</tt>,
+    # <tt>:constructed</tt>, or <tt>:special</tt>. See http://www.sil.org/iso639-3/types.asp
+    # for a description of these.
+    attr_reader :language_type
+
+    # Returns the reference language name.
+    attr_reader :reference_name
+
+    def initialize(identifier, part2b, part2t, part1, language_type, ref_name)
+      @identifier = identifier
+      @alpha3_bibliographic = part2b
+      @alpha3_terminology = part2t
+      @alpha2 = part1
+      @language_type = language_type
+      @reference_name = ref_name
+    end
+  end
+
+  class IndividualLanguage < Language
+    def initialize(individual_languages, macrolanguage, *rest)
+      super(*rest)
+
+      raise "Individual languages given for individual language" if individual_languages # FIXME
+      @macrolanguage = macrolanguage
+    end
+
+    # Returns the macrolanguage it is part of or +nil+ if not part of any
+    # macrolanguage.
+    def macrolanguage
+      ISOCodes::find_iso_639_3_language(@macrolanguage)
+    end
+  end
+
+  class Macrolanguage < Language
+    def initialize(individual_languages, macrolanguage, *rest)
+      super(*rest)
+
+      raise "Macrolanguage given for macrolanguage" if macrolanguage # FIXME
+      @individual_languages = individual_languages
+    end
+
+    # Returns an array of individual languages, otherwise an empty array.
+    def individual_languages
+      @individual_languages.map { |c| ISOCodes::find_iso_639_3_language(c) }
+    end
+  end
+
+  class << self
+    # Returns an object describing the language identified by the language
+    # code.
+    def find_language(code)
+      find_iso_639_3_language(code)
+    end
+
+    # Returns an object describing the language identified by an ISO 639-3
+    # identifier.
+    def find_iso_639_3_language(code)
+      if @@iso_639_3.has_key?(code)
+        klass, *rest = @@iso_639_3[code]
+        klass.new(*rest)
+      else
+        nil
+      end
+    end
+  end
+
+  private
+
+  LANGUAGE_TYPES = {
+    'L' => :living,
+    'E' => :extinct,
+    'A' => :ancient,
+    'H' => :historic,
+    'C' => :constructed,
+    'S' => :special,
+  }.freeze
+
+  DATA_PATH = File.expand_path(File.dirname(__FILE__))
+
+  class << self
+    def get_data_filename(filename)
+      File.join(DATA_PATH, filename)
+    end
+
+    def read_data_file(filename, field_count, delimiter = /,\s*/, skip_first = false)
+      Zlib::GzipReader.open(get_data_filename(filename)).each_line do |l|
+        if skip_first
+          skip_first = false
+          next
+        end
+
+        yield l.chomp.split(delimiter, field_count)
+      end
+    end
+
+    def load_iso_639_3
+      data = {}
+
+      read_data_file("iso-639-3_#{ISO_639_3_VERSION}.tab.gz", 8, "\t", true) do |args|
+        identifier, part2b, part2t, part1, scope, language_type, ref_name, comment = args
+
+        # Sanity checks
+        raise ArgumentError, "missing identifier" if identifier.nil?
+        raise ArgumentError, "missing reference name" if ref_name.nil?
+
+        case scope
+        when 'I': klass = IndividualLanguage
+        when 'M': klass = Macrolanguage
+        when 'S': klass = nil # FIXME
+        else
+          raise ArgumentError, "invalid scope"
+        end
+
+        language_type = LANGUAGE_TYPES[language_type]
+        raise ArgumentError, "invalid language type" if language_type.nil?
+
+        data[identifier] = [klass, nil, nil, identifier, part2b, part2t, part1, language_type, ref_name]
+      end
+
+      read_data_file("iso-639-3-macrolanguages_#{ISO_639_3_MACROLANGUAGE_MAPPINGS_VERSION}.tab.gz", 3, "\t", true) do |args|
+        macrolanguage_identifier, individual_language_identifier, status = args
+
+        case status
+        when 'R': next #FIXME
+        when 'A':
+        else
+          raise ArgumentError, "invalid status"
+        end
+
+        # Add macrolanguage to the individual language
+        raise "individual language already has a macrolanguage " if data[individual_language_identifier][2]
+        data[individual_language_identifier][2] = macrolanguage_identifier
+
+        # Add individual language to macrolanguage
+        data[macrolanguage_identifier][1] ||= []
+        data[macrolanguage_identifier][1] << individual_language_identifier
+      end
+
+      data.each_pair { |k, v| v.freeze }
+
+      data
+    end
+  end
+
+  @@iso_639_3 = load_iso_639_3
+end

data/test/test_iso_codes.rb

@@ -0,0 +1,35 @@
+require 'iso_codes'
+require 'test/unit'
+
+class ISO6393CodesTestCase < Test::Unit::TestCase
+  def test_find_language
+    l = ISOCodes::find_language("eng")
+    assert_not_nil l
+    assert_equal   'eng', l.identifier
+    assert_equal   'eng', l.alpha3_terminology
+    assert_equal   'eng', l.alpha3_bibliographic
+    assert_equal   'en', l.alpha2
+    assert_equal   'English', l.reference_name
+  end
+
+  def test_find_language_macrolanguage
+    l = ISOCodes::find_language("ara")
+    assert_not_nil l
+    assert_kind_of ISOCodes::Macrolanguage, l
+    assert         l.individual_languages.any? { |c| c.identifier == 'arq' }
+  end
+
+  def test_doc_exx
+    language = ISOCodes.find_language('lav')
+    assert_equal 'Latvian', language.reference_name
+    assert_equal ISOCodes::Macrolanguage, language.class
+    assert_equal 'lav', language.alpha3_bibliographic
+    assert_equal 'lv', language.alpha2
+
+    assert_equal ["ltg", "lvs"], language.individual_languages.map { |l| l.identifier }
+
+    language = ISOCodes.find_language('ltg')
+    assert_equal ISOCodes::IndividualLanguage, language.class
+    assert_equal "lav", language.macrolanguage.identifier
+  end
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification 
+name: iso-codes
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- "Marius L. J\xC3\xB8hndal"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-29 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: A database of ISO language and script codes.
+email: mariuslj (at) ifi [dot] uio (dot) no
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- README.todo
+files: 
+- .gitignore
+- CHANGELOG
+- README.rdoc
+- Rakefile
+- VERSION
+- iso-codes.gemspec
+- lib/iso-639-3-macrolanguages_20100128.tab.gz
+- lib/iso-639-3_20100330.tab.gz
+- lib/iso_codes.rb
+- test/test_iso_codes.rb
+- README.todo
+has_rdoc: true
+homepage: http://github.com/mlj/iso-codes
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: iso-codes
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: ISO language and script code support
+test_files: 
+- test/test_iso_codes.rb