lang 0.1.0.pre

data/README.rdoc

@@ -0,0 +1,46 @@
+== DESCRIPTION:
+
+Language tags implementation.
+
+== FEATURES:
+
+* RFC5646 conformance
+* Basic filtering (RFC 4647)
+* Extended filtering (RFC 4647)
+* Canonicalization
+* Direct work with IANA language subtag registry
+
+== EXAMPLES:
+
+  See examples directory:
+  http://github.com/SSDany/lang/tree/master/examples
+
+== INSTALLATION:
+
+  $ gem in lang
+  $ lang update
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2010
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/lang

@@ -0,0 +1,150 @@
+#!/usr/bin/env ruby
+
+dir = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$:.unshift(dir) unless $:.include?(dir)
+require 'lang/subtags'
+
+require 'net/http'
+require 'tempfile'
+
+module Lang
+  module Subtags
+    class Registry
+
+      NAME_REGEX = /^(?:#{SUBTAG}|#{TAG}):\s*([\w-]+)\s*$/io.freeze
+      TYPE_REGEX = /^#{TYPE}:\s*(\w+)\s*$/io.freeze
+
+      def initialize(path)
+        @path = File.expand_path(path)
+      end
+
+      def exists?
+        File.exists?("#{@path}.registry")
+      end
+
+      def download(uri)
+        FileUtils.mkdir_p(File.dirname(@path)) unless exists?
+        write("registry") { |temp| http(uri) { |chunk| temp << chunk }}
+      end
+
+      def build_indices
+        return false unless exists?
+
+        STDOUT << "Building indices\n"
+        calculate_indices
+        calculate_boundaries
+
+        write("indices") do |temp|
+          @boundaries.each do |boundary|
+            template = "%-#{boundary[-2]}s%#{boundary[-1] - boundary[-2] - 1}d\n"
+            @indices[boundary.first].to_a.sort.each { |k,v| temp << template % [k,v] }
+          end
+        end
+
+        write("boundaries") do |temp|
+          @boundaries.each do |boundary|
+            temp << "#{boundary.join(":")}\n"
+          end
+        end
+
+        STDOUT << "Done\n"
+        true
+      end
+
+      private
+
+      def write(dest, &block)
+
+        path = "#{@path}.#{dest}"
+        temp = Tempfile.new(dest)
+        temp.binmode
+        yield(temp) if block_given?
+        temp.close
+
+        # somewhat stolen from ActiveSupport
+
+        begin
+          old = File.stat(path)
+        rescue Errno::ENOENT
+          check = File.join(File.dirname(path), ".permissions_check.#{Thread.current.object_id}.#{Process.pid}.#{rand(1000000)}")
+          File.open(check, File::WRONLY | File::CREAT) { }
+          old = File.stat(check)
+          File.unlink(check)
+        end
+
+        FileUtils.mv(temp.path, "#{@path}.#{dest}")
+
+        File.chown(old.uid, old.gid, path)
+        File.chmod(old.mode, path)
+        nil
+      end
+
+      def http(uri)
+        STDOUT << "Downloading #{uri}\n"
+        Net::HTTP.get_response(URI(uri)) do |response|
+          total, size = response['Content-Length'].to_i, 0
+          response.read_body do |chunk|
+            size += chunk.size
+            yield(chunk) if block_given?
+            STDOUT << "\r%d%% done (%d of %d)" % [size*100/total, size, total]
+            STDOUT.flush
+          end
+        end
+        STDOUT << "\n"
+        nil
+      end
+
+      def calculate_boundaries
+        calculate_indices unless @indices
+        offset = 0
+        @boundaries = @indices.keys.sort{ |a,b| a.to_s <=> b.to_s }.map do |kind|
+          segment   = @indices[kind]
+          boundary  = []
+          boundary << kind
+          boundary << offset
+          boundary << segment.size - 1
+          boundary << segment.keys.map{ |s| s.size }.max
+          boundary << segment.values.max.to_s.size + boundary.last + 1
+          offset   += segment.size * boundary.last
+          boundary
+        end
+        true
+      end
+
+      def calculate_indices
+        count = 0
+        kind, name = nil, nil
+        @indices = {}
+        File.open("#{@path}.registry", File::RDONLY) do |f|
+          f.each_line do |l|
+            if TYPE_REGEX === l
+              kind = $1.to_sym
+              @indices[kind] ||= {}
+            elsif kind && NAME_REGEX === l
+              name = $1.downcase
+              @indices[kind][name] = count
+            elsif l == SEPARATOR
+              kind, name = nil, nil
+            end
+            count += l.size
+          end
+        end
+        #STDOUT << "#{count}\n"
+        true
+      end
+
+    end
+  end
+end
+
+command = ARGV.shift
+unless %w(reindex update).include?(command)
+  STDERR << "unknown command: #{command.inspect}\n"
+  exit 1
+end
+
+registry = Lang::Subtags::Registry.new(ARGV.shift || Lang::Subtags.registry_path)
+registry.download("http://www.iana.org/assignments/language-subtag-registry") if command == 'update' || !registry.exists?
+registry.build_indices
+
+# EOF

data/lib/lang/subtags.rb

@@ -0,0 +1,147 @@
+require 'thread'
+require 'lang/subtags/entry'
+require 'lang/subtags/language'
+require 'lang/subtags/extlang'
+require 'lang/subtags/script'
+require 'lang/subtags/region'
+require 'lang/subtags/variant'
+require 'lang/subtags/grandfathered'
+require 'lang/subtags/redundant'
+
+module Lang #:nodoc:
+  module Subtags
+
+    LOCK            = Mutex.new
+    SEPARATOR       = "%%\n".freeze
+    TYPE            = "Type".freeze
+    SUBTAG          = "Subtag".freeze
+    TAG             = "Tag".freeze
+    ADDED           = "Added".freeze
+    DEPRECATED      = "Deprecated".freeze
+    DESCRIPTION     = "Description".freeze
+    COMMENTS        = "Comments".freeze
+    PREFIX          = "Prefix".freeze
+    PREFERRED_VALUE = "Preferred-Value".freeze
+    MACROLANGUAGE   = "Macrolanguage".freeze
+    SCOPE           = "Scope".freeze
+    SUPPRESS_SCRIPT = "Suppress-Script".freeze
+    CONTINUE_REGEX  = /\A\s\s/.freeze
+
+    COLON           = ":".freeze
+    COLON_SPLITTER  = RUBY_VERSION < '1.9.1' ? /\:/.freeze : COLON
+
+    SYM2CLASS = {}
+    Entry.subclasses.each do |subclass|
+      meth = subclass.to_s.gsub(/^.*::/,'')
+      kind = meth.downcase.to_sym
+      SYM2CLASS[kind] = subclass
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+      def #{meth}(s)
+        entry(:#{kind},s)
+      end
+      EOS
+    end
+
+    def entry(kind, snippet)
+      return nil unless SYM2CLASS.include?(kind)
+      klass = SYM2CLASS[kind]
+      LOCK.synchronize {
+        if klass.entries.key?(snippet) ||
+           klass.entries.key?(snippet = snippet.downcase)
+          return klass.entries[snippet]
+        end
+        klass.entries[snippet] = load_entry(kind, snippet)
+      }
+    end
+
+    def close
+      LOCK.synchronize {
+        registry.close
+        indices.close
+      }
+    end
+
+    def search(kind, snippet)
+
+      lower = 0
+      offset, upper, t, r = *boundaries[kind]
+      target = snippet.ljust(t)
+
+      until upper < lower
+        middle = (lower+upper)/2
+        indices.seek(offset + middle*r, IO::SEEK_SET)
+        value = indices.read(t)
+        if value == target
+          return indices.read(r-t).to_i
+        elsif target < value
+          upper = middle-1
+        else
+          lower = middle+1
+        end
+      end
+      nil
+    end
+
+    def load_entry(kind, snippet)
+      amount = search(kind, snippet)
+      return nil unless amount
+      registry.seek(amount, IO::SEEK_SET)
+      thing = SYM2CLASS[kind].new
+      until registry.eof? || registry.readline == SEPARATOR
+
+        line = $_
+        thing.comments << $' && next if CONTINUE_REGEX === line
+        attribute, value = line.split(COLON_SPLITTER,2)
+        value.strip!
+
+        case attribute
+        when DESCRIPTION      ; thing.add_description(value)
+        when PREFIX           ; kind == :variant ? thing.add_prefix(value) : thing.prefix = value
+        when SUBTAG,TAG       ; thing.name            = value
+        when ADDED            ; thing.added_at        = value
+        when DEPRECATED       ; thing.deprecated_at   = value
+        when COMMENTS         ; thing.comments        = value
+        when PREFERRED_VALUE  ; thing.preferred_value = value
+        when MACROLANGUAGE    ; thing.macrolanguage   = value
+        when SCOPE            ; thing.scope           = value
+        when SUPPRESS_SCRIPT  ; thing.suppress_script = value
+        end
+
+      end
+      thing
+    end
+
+    def registry_path
+      @registry_path ||= File.join(File.dirname(__FILE__), "data", "language-subtag")
+    end
+
+    def registry
+      @registry ||= File.open("#{registry_path}.registry", File::RDONLY)
+    end
+
+    def indices
+      @indices ||= File.open("#{registry_path}.indices", File::RDONLY)
+    end
+
+    def boundaries
+      return @boundaries if @boundaries
+      @boundaries = {}
+      File.open("#{registry_path}.boundaries", File::RDONLY).each_line do |line|
+        boundary = line.split(COLON_SPLITTER)
+        @boundaries[boundary.shift.to_sym] = boundary.map { |b| b.to_i }
+      end
+      @boundaries
+    end
+
+    extend self
+
+    class << self
+      private :boundaries, :indices, :registry
+      private :load_entry
+      private :search
+    end
+
+  end
+end
+
+# EOF

data/lib/lang/subtags/entry.rb

@@ -0,0 +1,40 @@
+module Lang #:nodoc:
+  module Subtags
+    class Entry
+
+      attr_accessor :name,
+                    :preferred_value,
+                    :added_at,
+                    :deprecated_at,
+                    :comments
+
+      def deprecated?
+        !@deprecated_at.nil?
+      end
+
+      def description
+        @descriptions.join("\n") if @descriptions
+      end
+
+      def add_description(chunk)
+        @descriptions ||= []
+        @descriptions << chunk
+      end
+
+      def self.inherited(subclass)
+        subclasses << subclass
+      end
+
+      def self.subclasses
+        @subclasses ||= []
+      end
+
+      def self.entries
+        @entries ||= {}
+      end
+
+    end
+  end
+end
+
+# EOF

data/lib/lang/subtags/extlang.rb

@@ -0,0 +1,19 @@
+module Lang #:nodoc:
+  module Subtags
+    # Holds data about extlang subtags.
+    class Extlang < Entry
+
+      attr_accessor :macrolanguage,
+                    :suppress_script,
+                    :prefix,
+                    :scope
+
+      def macro
+        Subtags.entry(:language, macrolanguage) if macrolanguage
+      end
+
+    end
+  end
+end
+
+# EOF

data/lib/lang/subtags/grandfathered.rb

@@ -0,0 +1,9 @@
+module Lang #:nodoc:
+  module Subtags
+    # Holds data about grandfathered registrations.
+    class Grandfathered < Entry
+    end
+  end
+end
+
+# EOF

data/lib/lang/subtags/language.rb

@@ -0,0 +1,18 @@
+module Lang #:nodoc:
+  module Subtags
+    # Holds data about primary language subtags.
+    class Language < Entry
+
+      attr_accessor :macrolanguage,
+                    :suppress_script,
+                    :scope
+
+      def macro
+        Subtags.entry(:language, macrolanguage) if macrolanguage
+      end
+
+    end
+  end
+end
+
+# EOF

data/lib/lang/subtags/redundant.rb

@@ -0,0 +1,9 @@
+module Lang #:nodoc:
+  module Subtags
+    # Holds data about redundant tags.
+    class Redundant < Entry
+    end
+  end
+end
+
+# EOF