langtag 0.1.0

data/README

@@ -0,0 +1,31 @@
+== Langtag version 0.1.0
+
+=== Overview
+The Langtag class represents IETF Language Tags
+as a subclass of String that provides read/write
+access to specific parts of a language tag, such
+as language, script, region, and so on, as well
+as well-formedness checks and some other simple
+operations.
+
+=== IETF Language Tags
+IETF Language Tags are defined by BCP 47
+(http://www.ietf.org/rfc/bcp/bcp47.txt),
+which currently consists of RFC 4646
+(http://www.ietf.org/rfc/rfc4646.txt)
+and RFC 4647 (http://www.ietf.org/rfc/rfc467.txt).
+These documents are the work of the LTRU Working
+Group (see http://www.ietf.org/html.charters/ltru-charter.html).
+For further explanatory information on IETF language tags,
+in particular in a Web context, please also see
+http://www.w3.org/International/articles/language-tags/.
+
+=== Future Work
+- Provide support for various matching/lookup options
+  defined in RFC 4647
+- Provide support for validation
+
+=== Copyright
+Copyright (c) 2007 Martin J. Du"rst (duerst@it.aoyama.ac.jp)
+Licensed under the same terms as Ruby. Absolutely no warranty.
+(see http://www.ruby-lang.org/en/LICENSE.txt)

data/lib/langtag.rb

@@ -0,0 +1,170 @@
+# :include: ../README
+
+# :stopdoc:
+# === Helper Functions for Array Class
+class Array
+  # Create a new Array by repeatedly calling &block
+  # until the returned value is equal to final.
+  def Array.collect (final=nil, &block)
+    a = Array.new
+    while final != (e=block.call)
+      a << e
+    end
+    a
+  end
+  # Test uniqueness of elements of an Array.
+  # This should be available as part of the standard Array class.
+  def uniq?
+    return self.length == self.uniq.length
+  end
+end
+# :startdoc:
+
+require 'rubygems'
+gem 'composite'
+require 'composite'
+
+# Langtag class, implementing BCP 47 (currently RFC 4646)
+# IETF language tags. Provides decomposition of language
+# tags into components, and wellformedness check.
+#
+# ==== Accessor methods
+# Getting: language, script, region, variants, extensions, private.
+# Setting: language=, script=, region=, variants=, extensions=, private=.
+# Variants and extensions accessors get/set Arrays, the other accessors get/set Strings.
+# Because of the way Ruby assignement methods are implemented,
+# manipulating variants and extensions with e.g.
+#   myLangtag.variants += ['e-Extension']
+# (adding 'e-Extension' as an extension to whatever extensions myLangtag
+#  already has) is possible. Similarly,
+#   myLangtag.variants -= ['e-Extension']
+# will again remove the extension.
+class Langtag < String
+  include Composite
+  def initialize (s)
+    super(s)
+    decompose
+  end
+
+  # the parts of a language tag
+  # variants and extensions are arrays, the other parts are strings
+  part :language, :script, :region, :variants, :extensions, :private
+  
+  # Array of irregular language tags
+  Irregular = ['en-gb-oed',
+     'i-ami', 'i-bnn', 'i-default', 'i-enochian', 'i-hak',
+     'i-klingon', 'i-lux', 'i-mingo', 'i-navajo', 'i-pwn',
+     'i-tao', 'i-tay', 'i-tsu', 'sgn-be-fr',
+     'sgn-be-nl', 'sgn-ch-de'];
+  
+  # Array of grandfathered language tags
+  Grandfathered = Irregular + ['art-lojban', 'cel-gaulish',
+     'no-bok', 'no-nyn', 'zh-cmn', 'zh-cmn-hans', 'zh-cmn-hant',
+     'zh-gan', 'zh-guoyu', 'zh-hakka', 'zh-min', 'zh-min-nan',
+     'zh-wuu', 'zh-xiang', 'zh-yue'];
+
+  # returns true if language tag is well-formed, false otherwise
+  def wellformed? ()
+    decompose
+    @wellformed
+  end
+
+  # returns true if language tag is grandfathered, false otherwise
+  def grandfathered? ()
+    Grandfathered.include? self.to_str.downcase
+  end
+
+  # returns true if language tag is irregular, false otherwise
+  def irregular? ()
+    Irregular.include? self.to_str.downcase
+  end
+
+  # changes case to look 'nice' (regions are UPPER-CASE,
+  # scripts are Title-Case, everything else is lower case
+  def nicecase! ()
+    @language.downcase!
+    @script.capitalize!
+    @region.upcase!
+    @variants.each { |v| v.downcase! }
+    @extensions.each { |e| e.downcase! }
+    @private.downcase!
+    compose
+  end
+
+  # non-descructive variant of nicecase!: returns a nicecased copy
+  def nicecase ()
+    result = Langtag.new(self).nicecase!
+  end
+
+  # compose the langtag from parts, joining with '-'
+  # flatten first to deal with @variants/@extentsions with are arrays
+  # then compact to remove nil values (mainly internal use)
+  def compose
+    replace([@language, @script, @region, @variants,
+             @extensions, @private].flatten.compact.join('-'))
+  end
+
+  # decompose a language tag into parts (mainly internal use)
+  def decompose
+    # check if we really need to decompose again
+    if @saved == self.to_str
+      return
+    end
+    # initialize everything
+    s = @saved = self.to_str # save for check next time around
+    @wellformed = true   # assume well-formed
+    @language = @script = @region = @private = nil
+    @variants = @extensions = []
+
+    # deal with irregular and completely private langtags
+    if irregular? || s =~ /^x-/i
+      @language = s
+      return
+    end
+    # check well-formedness with a single regular expression,
+    # except for irregulars (checked above) and multiple
+    # occurrences of the same extension (checked below)
+    # notice /i modifier for case insensitive matching
+    if not(s =~ /^([a-z]{2,3}                       # shortest ISO 639 language
+                    (-[a-z]{3}){0,3}                  # with optional extensions
+                   |[a-z]{4,8})                       # or reserved\registered
+                  (-[a-z]{4})?                      # optional script
+                  (-([a-z]{2}|\d{3}))?              # optional region
+                  (-([a-z0-9]{5,8}|\d[a-z0-9]{3}))* # optional variants
+                  (-[a-wyz0-9](-[a-z0-9]{2,8})+)*   # optional extensions
+                  (-x(-[a-z0-9]{1,8})+)?            # optional private use part
+                  $/ix)
+      @wellformed = false
+    end
+    # extract language
+    if s =~ /^(([a-z]{2,3}(-[a-z]{3}){0,3}|[a-z]{4,8}))(-|$)/i
+      @language, s = $1, $'
+    end
+    # extract private use tail
+    if s =~ /(^|-)(x-.*)$/i
+      s, @private = $`, $2
+    end
+    # extract extensions and check for duplicates
+    @extensions = Array.collect do
+      if s =~ /(^|-)([a-wyz0-9](-[a-z0-9]{2,8})+)$/i
+        s = $`
+        $2
+      else
+        nil
+      end
+    end
+    @extensions.reverse! # put back in order
+    if !((@extensions.collect {|ext| ext[0..1].downcase}).uniq?)
+      @wellformed = false
+    end
+    if s =~ /(^|-)([a-z]{4})(-|$)/i    # extract script
+      @script = $2
+    end
+    if s =~ /(^|-)([a-z]{2}|\d{3})(-|$)/i    # extract region
+      @region = $2
+    end
+    # extract variants
+    @variants = s.scan(/(^|-)([a-z0-9]{5,8}|\d[a-z0-9]{3})(?=(-|$))/i).
+                  collect { |match| match[1] }
+  end # decompose
+end # class Langtag

data/test/langtagTest.txt

@@ -0,0 +1,309 @@
+WELL-FORMED
+
+AaBbCcDd
+AaBbCcDd-x-y-any-x
+abcd-Latn
+ab-x-abc-a-a   	 # ditto
+ab-x-abc-a-a 	 # ditto
+ab-x-abc-x-abc 	 # anything goes after x
+ab-x-abc-x-abc 	 # anything goes after x
+ax-TZ 	 # Not in the registry, but well-formed
+az-Arab-x-AZE-derbend
+de-a-value
+de-CH-1996
+de-Latg-1996
+en
+en-a-bbb-x-a-ccc
+en-enx
+en-enx-eny-enz-latn-us
+en-gb-oed
+en-Latn
+en-Latn-001
+en-Latn-GB-boont-r-extended-sequence-x-private
+en-Latn-US
+en-Latn-US-lojban-gaulish
+en-Latn-US-lojban-gaulish-a-12345678-ABCD-b-ABCDEFGH
+en-Latn-US-lojban-gaulish-a-12345678-ABCD-b-ABCDEFGH-x-a-b-c-12345678
+en-US
+en-US
+en-US-boont
+en-x-US
+es-419
+es-Latn-CO-x-private
+fr 
+fra
+fra-FX
+fra-Latn 	 # ISO 639 can be 3-letters
+fr-FR
+fr-fra 	 # Extended tag
+fr-Lat 	 # Extended
+fr-Latn
+fr-Latn-419
+fr-Latn-CA
+fr-Latn-CA
+fr-Latn-FR
+fr-shadok 	 # Variant
+fr-y-myext-myext2
+i-default      	 # grandfathered
+i-default 	 # grandfathered
+i-enochian 	 # Grand fathered
+i-klingon 	 # grandfathered
+i-klingon 	 # grandfathered with singleton
+mn-Cyrl-MN
+mN-cYrL-Mn
+no-bok 	 # grandfathered without singleton
+sl-IT-nedis
+sl-nedis
+sr-Latn-CS
+x-12345678-a
+x-fr-CH
+
+#all of the grandfathered codes; first the really goofy ones. Cased oddly for an extra test.
+En-Gb-Oed
+I-Ami
+I-Bnn
+I-Default
+I-Enochian
+I-Hak
+I-Klingon
+I-Lux
+I-Mingo
+I-Navajo
+I-Pwn
+I-Tao
+I-Tay
+I-Tsu
+Sgn-Be-Fr
+Sgn-Be-Nl
+Sgn-Ch-De
+
+#now the ones that are well-formed, but currently invalid
+art-lojban
+cel-gaulish
+en-boont
+en-scouse
+no-bok
+no-nyn
+zh-cmn
+zh-cmn-Hans
+zh-cmn-Hant
+zh-gan
+zh-guoyu
+zh-hakka
+zh-min
+zh-min-nan
+zh-wuu
+zh-xiang
+zh-yue
+
+# Now some randomly generated correct names
+
+cfR-wOG-g-UkjoqWt8-ii8S04LL-rbBDq0gl-o-qmzs-ifnRSqVz-241T-lVFJq30L-0JWuHsb-C-WMThK-kbEOuA-tIQ-Lfjt-a-c1gdojdJ-7iv-b-NawXDK
+JP-ubE-JtS-fOa-BOiO
+BYE-fiX-mKH-BKdy
+rc-Ajl-jpl-X-Lh-SPB-ANEXM
+FwtsUTb
+GH-NgZ-rW
+x-GF-E7m-v2-V09q
+qQ-THL-dth
+TDGbw
+Uif-eE
+TDJVhlwx-HEwn-6M9a-4DvI-7WaG-8IRj-7QEk-7yID
+rOO-yUE-UEY-bbcM
+yY-jDQ-eDK-NsZ-a-oJQ-eLc-JTMc31-nhr-h-unOGj8-Os7-JMT5jeVq-pE-QK5
+oJ-57b1OPWP-6MUqYs-Kesk65J-74oa-34Ys
+vX-PCm-mnT-PsQ
+eQh-ewFo-7mvP-EbJ0xx-0OEK-I0I8ju6
+JwdjKtH-WD
+QxxLG-X-lJNNX6Pu-7OfhbLoa
+icb-AAk-EFU-dAyc-1rj2-3upg-8VMx-5nml-DF45sBf
+en-GB-oed
+oHSgh-x-SpoWfaO8-J-NATtSLZ1
+Ehl-hpi-Bbb-zu-x-n6rrcoz0
+UFhQ-GO-X-HULOlod-tkv
+jVVxZr
+HkFqh
+tyg-mMk-YME-Nia-aJab-Ej
+rI-aqM-gkp-ZNW-NW
+Hv-acb-248
+XMXIUasy-Djzc-eW
+DGKW-oZRC-G-1P-SuP-A-GI2SuR-vX-rnH1Y8-heft
+X-C5lAw-Hn-XdR7x
+tsQpkxE-xwuk
+i-pwn
+SQrky-AWCe-xo-x-cU-fo2-u1KhUJ
+aNCzqvs-IVeQ-ZY
+sj
+jeMSw
+UPm-sMd-Dn
+Hho-sG-GpcoS1-IxGcI
+EDrfxBz
+qpW-HBWu-ta
+AIl-FGV
+Lqn-bid-DpI
+Jm
+xE-Lxs-qu
+RliJDAg
+ct-gwQ-SIu
+csneMbEX-Umid-r-AfHD-gDWov-DfxmF4ew-0ENgU-S-pBN9O4c-9HK-c0ElsKnC
+jH-BIYY-pT
+en-GB-oed
+
+ILL-FORMED
+
+-a
+a-
+a1-Hant-ZH
+aabbccddE
+a--b
+ab-123-abc
+ab-123-abc
+ab-123-abcd
+ab-123-abcd
+ab-1abc-abc
+ab-1abc-abc
+ab-1abc-abcd
+ab-1abc-abcd
+ab--ab
+ab--ab
+ab-a-b
+ab-a-b
+ab-ab-abc
+ab-ab-abc
+ab-ab-abcd
+ab-ab-abcd
+-ab-abc
+-ab-abc
+ab-abc-
+ab-abc-
+ab-abc-abc-abc-abc
+ab-abc-abc-abc-abc
+ab-abcd-abc
+ab-abcd-abc
+ab-abcde-abc
+ab-abcde-abc
+ab-abcde-abcd
+ab-abcde-abcd
+ab-a-x
+ab-a-x
+abcd-efg
+abcdefghi-012345678
+abcdefghi-012345678
+a-foo
+a-Hant-ZH
+a-value
+a-x
+b-fish
+en-enx-eny-enz-enw
+en-UK-oed
+en-US-Latn
+f
+f-Latn
+fr-Latn-F
+overlongone
+tlh-a-b-foo
+
+i-notexist          	 # grandfathered but not registered: invalid, even if we only test well-formedness
+
+# the following have multiple singletons
+ab-a-abc-a-abc
+en-a-bbb-a-ccc	 # 'a' appears twice 
+ab-c-abc-r-toto-c-abc  	 # 'c' appears twice 
+
+#mechanically generated ill-formed items
+EdY-z_H791Xx6_m_kj
+qWt85_8S0-L_rbBDq0gl_m_O_zsAx_nRS
+VzyL2
+T_VFJq-L-0JWuH_u2_VW-hK-kbE
+u-t
+Q-f_ZVJXyc-doj_k-i
+JWB7gNa_K-5GB-25t_W-s-ZbGVwDu1-H3E
+b-2T-Qob_L-C9v_2CZxK86
+fQTpX_0_4Vg_L3L_g7VtALh2
+S-Z-E_J
+f6wsq-02_i-F
+9_GcUPq_G
+QjsIy_9-0-7_Dv2yPV09_D-JXWXM
+D_se-f-k
+ON47Wv1_2_W
+f-z-R_s-ha
+N3APeiw_195_Bx2-mM-pf-Z-Ip5lXWa-5r
+IRjxU-E_6kS_D_b1b_H
+NB-3-5-AyW_FQ-9hB-TrRJg3JV_3C
+yF-3a_V_FoJQAHeL_Z-Mc-u
+n_w_bbunOG_1-s-tJMT5je
+Q-AEWE_X
+57b1O_k_R6MU_sb
+hK_65J_i-o_SI-Y
+wB4B7u_5I2_I_NZPI
+J24Nb_q_d-zE
+v6-dHjJmvPS_IEb-x_A-O-i
+8_8_dl-ZgBr84u-P-E
+nIn-xD7EVhe_C
+5_N-6P_x7Of_Lo_6_YX_R
+0_46Oo0sZ-YNwiU8Wr_d-M-pg1OriV
+laiY-5
+K-8Mdd-j_ila0sSpo_aO8_J
+wNATtSL-Cp4_gPa_fD41_9z
+H_FGz5V8_n6rrcoz0_1O6d-kH-7-N
+wDOrnHU-odqJ_vWl
+gP_qO-I-jH
+h
+dJ0hX-o_csBykEhU-F
+L-Vf7_BV_eRJ5goSF_Kp
+y-oF-chnavU-H
+9FkG-8Q-8_v
+W_l_NDQqI-O_SFSAOVq
+kDG3fzXw
+t-nsSp-7-t-mUK2
+Yw-F
+1-S_3_l
+u-v_brn-Y
+4_ft_3ZPZC5lA_D
+n_dR-QodsqJnh_e
+Hwvt-bSwZwj_KL-hxg0m-3_hUG
+mQHzvcV-UL-o2O_1KhUJQo_G2_uryk3-a
+b-UTn33HF
+r-Ep-jY-aFM_N_H
+K-k-krEZ0gwD_k_ua-9dm3Oy-s_v
+XS_oS-p
+EIx_h-zf5
+p_z-0_i-omQCo3B
+1_q0N_jo_9
+0Ai-6-S
+L-LZEp_HtW
+Zj-A4JD_2A5Aj7_b-m3
+x
+p-qPuXQpp_d-jeKifB-c-7_G-X
+X94cvJ_A
+F2D25R_qk_W-w_Okf_kx
+rc-f
+D
+gD_WrDfxmF-wu-E-U4t
+Z_BN9O4_D9-D_0E_KnCwZF-84b-19
+T-8_g-u-0_E
+lXTtys9j_X_A_m-vtNiNMw_X_b-C6Nr
+V_Ps-4Y-S
+X5wGEA
+mIbHFf_ALu4_Jo1Z1
+ET-TacYx_c
+Z-Lm5cAP_ri88-d_q_fi8-x
+rTi2ah-4j_j_4AlxTs6m_8-g9zqncIf-N5
+FBaLB85_u-0NxhAy-ZU_9c
+x_j_l-5_aV95_s_tY_jp4
+PL768_D-m7jNWjfD-Nl_7qvb_bs_8_Vg
+9-yOc-gbh
+6DYxZ_SL-S_Ye
+ZCa-U-muib-6-d-f_oEh_O
+Qt-S-o8340F_f_aGax-c-jbV0gfK_p
+WE_SzOI_OGuoBDk-gDp
+cs-Y_9
+m1_uj
+Y-ob_PT
+li-B
+f-2-7-9m_f8den_J_T_d
+p-Os0dua-H_o-u
+L
+rby-w
+
+

data/test/test_langtag.rb

@@ -0,0 +1,78 @@
+# some unit tests for the Langtag class
+
+# Copyright 2007 Martin J. Du"rst (duerst@it.aoyama.ac.jp);
+# available under the same licence as Ruby itself
+# (see http://www.ruby-lang.org/en/LICENSE.txt)
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'langtag'
+
+require 'test/unit'
+class TestLangtag < Test::Unit::TestCase
+  # Test to make sure that for a language tag starting with x-,
+  # the whole thing is taken as a language part, rather than as a private part
+  # (we are using the principle that every (well-formed) language tag
+  # has to have a language part).
+  def test_private_only
+    s = Langtag.new('x-just-private-only')
+    assert_equal('x-just-private-only', s.language)
+    assert_equal(nil, s.script)
+    assert_equal(nil, s.region)
+    assert_equal([], s.variants)
+    assert_equal([], s.extensions)
+    assert_equal(nil, s.private) # this may be somewhat surprising
+    assert_equal(true, s.wellformed?)
+  end
+  
+  # Test a long language tag
+  # Shows some uses of += and -= to manipulate variants and extensions.
+  def test_long
+    s = Langtag.new('de-Latn-ch-p-abc-q-def-x-myself')
+    assert_equal('de', s.language)
+    assert_equal('Latn', s.script)
+    assert_equal('ch', s.region)
+    assert_equal([], s.variants)
+    assert_equal(['p-abc', 'q-def'], s.extensions)
+    assert_equal('x-myself', s.private)
+    s.variants += ['fonipa']
+    assert_equal(['fonipa'], s.variants)
+    assert_equal('de-Latn-ch-fonipa-p-abc-q-def-x-myself', s)
+    assert_equal(true, s.wellformed?)
+    s.extensions += ['p-again']
+    assert_equal(false, s.wellformed?)
+    s.extensions -= ['p-abc']
+    assert_equal(true, s.wellformed?)
+    assert_equal(['q-def', 'p-again'], s.extensions)
+  end
+  
+  # Some nice-casing tests
+  def test_nice
+    s = Langtag.new('DE-LATN-CH-P-ABC-Q-DEF-X-MYSELF')
+    s.nicecase!
+    assert_equal('de-Latn-CH-p-abc-q-def-x-myself', s)
+    s = Langtag.new('de-latn-ch-p-abc-q-def-x-myself')
+    assert_equal('de-Latn-CH-p-abc-q-def-x-myself', s.nicecase)
+    assert_equal('Latn', s.nicecase.script)    
+  end
+
+  # Mechanical tests for well-formedness, using file langtagTest.txt downloaded from
+  # http://unicode.org/cldr/data/tools/java/org/unicode/cldr/util/data/langtagTest.txt,
+  # which includes all kinds of tests, many of them collected from the
+  # ltru@ietf.org mailing list.
+  def test_langtagTest
+    wellformed = false # keep track of range in file
+    File.open(File.join(File.dirname(__FILE__), 'langtagTest.txt')) do |file|
+      file.each_with_index do |line, i|
+        tag = line.chomp.sub(/\#.*/, '').strip
+        if tag == 'WELL-FORMED'
+          wellformed = true
+        elsif tag == 'ILL-FORMED'
+          wellformed = false
+        elsif tag != ''
+          assert_equal(wellformed, Langtag.new(tag).wellformed?,
+                       'langtagTest.txt, line: '+(i+1).to_s+'; tag: '+tag)
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.2
+specification_version: 1
+name: langtag
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-03-27 00:00:00 +09:00
+summary: "Support for IETF Language Tags (BCP 47, currently RFC 4646): Wellformedness check, read/write access to parts such as language, script, region, etc."
+require_paths: 
+- lib
+email: duerst@it.aoyama.ac.jp
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: langtag
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Martin J. Du"rst
+files: 
+- lib/langtag.rb
+- test/langtagTest.txt
+- test/test_langtag.rb
+- README
+test_files: 
+- test/test_langtag.rb
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: composite
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.3.0
+    version: