localite 0.3 → 0.5.6

data/lib/localite/storage.rb

@@ -0,0 +1,132 @@
+require 'i18n'
+require 'stringio'
+require 'etest'
+
+module Localite
+  module Backend; end
+end
+
+require "#{File.dirname(__FILE__)}/tr"
+
+class Localite::Backend::Simple < I18n::Backend::Simple
+  def initialize(*args)
+    @locales = args.map(&:to_s) unless args.empty?
+  end
+
+  def available_locales
+    if @locales 
+      @locales.map(&:to_sym)
+    else
+      super
+    end
+  end
+
+  #
+  # Loads a single translations file by delegating to #load_rb or
+  # #load_yml depending on the file extension and directly merges the
+  # data to the existing translations. Raises I18n::UnknownFileType
+  # for all other file extensions.
+  def load_file(filename)
+    locale_from_file = File.basename(filename).sub(/\.[^\.]+$/, "")
+    if @locales && !@locales.include?(locale_from_file)
+      # dlog "Skipping translations from #{filename}"
+      return
+    end
+
+    Localite.logger.warn "Loading translations from #{filename}"
+    type = File.extname(filename).tr('.', '').downcase
+    raise I18n::Backend::Simple::UnknownFileType.new(type, filename) unless respond_to?(:"load_#{type}")
+
+    data = send :"load_#{type}", filename
+
+    #
+    # 
+    if locale_from_file.length == 2 && data.keys.map(&:to_s) != [ locale_from_file ]
+      merge_translations(locale_from_file, data) 
+    else
+      data.each { |locale, d| merge_translations(locale, d) }
+    end
+  end
+
+  #
+  # add an additional translation storage layer.
+  def merge_translations(locale, data)
+    translations_for_locale!(locale).update data
+    super
+  rescue IndexError
+  end
+  
+  def translations
+    @translations ||= {}
+  end
+
+  def keys_for_locale(locale)
+    (translations[locale.to_sym] || {}).keys.sort
+  end
+
+  def keys
+    r = []
+    translations.each do |k,v|
+      r += v.keys
+    end
+    r.sort.uniq
+  end
+  
+  def translations_for_locale(locale)
+    translations[locale.to_sym]
+  end
+
+  def translations_for_locale!(locale)
+    translations[locale.to_sym] ||= {}
+  end
+  
+  #
+  # monkeypatches "I18n::Backend::Simple"
+  #
+  # Looks up a translation from the translations hash. Returns nil if
+  # eiher key is nil, or locale, scope or key do not exist as a key in the
+  # nested translations hash. Splits keys or scopes containing dots
+  # into multiple keys, i.e. <tt>currency.format</tt> is regarded the same as
+  # <tt>%w(currency format)</tt>.
+  #
+  # We are doing this because the default simple storage does not support
+  # keys, that inself are part of other keys, i.e. "field" and "field.subfield"
+  def lookup(locale, key, scope = [], options = {})
+    return unless key
+    init_translations unless initialized?
+
+    keys = I18n.normalize_keys(locale, key, scope, options[:separator])
+    
+    #
+    # the first key is the locale; all other keys make up the final key.
+    lookup_localite_storage(*keys) || begin
+      keys.inject(translations) do |result, key|
+        key = key.to_sym
+        return nil unless result.is_a?(Hash) && result.has_key?(key)
+        result = result[key]
+        result = resolve(locale, key, result, options) if result.is_a?(Symbol)
+        String === result ? result.dup : result
+      end
+    end
+  end
+
+  def lookup_localite_storage(locale, *keys)
+    return nil unless tr = translations_for_locale(locale)
+    r = tr[keys.join(".")]
+  end
+    
+  #
+  # The main differences to a yaml file are: ".tr" files support 
+  # specific and lower level entries for the same key, and allows
+  # to reopen a key.
+  #
+  # x:
+  #   y: "x.y translation"
+  #     z: "x.y.z translation"
+  #   y: 
+  #     a: "x.y.a translation"
+  #
+  def load_tr(filename)
+    Localite::Backend::Tr.load(filename)
+  end
+end

data/lib/localite/template.rb

@@ -1,8 +1,16 @@
 require "cgi"
 
 class Localite::Template < String
-  def self.run(mode, template, opts = {})
-    new(template).run(mode, opts)
+  def self.run(template, opts = {})
+    new(template).run(Localite.current_format, opts)
+  end
+
+  def self.html(template, opts = {})
+    new(template).run(:html, opts)
+  end
+
+  def self.text(template, opts = {})
+    new(template).run(:text, opts)
   end
 
   #
@@ -65,7 +73,8 @@ class Localite::Template < String
     end
   end
   
-  def run(mode, opts = {})
+  def run(format, opts = {})
+
     #
     env = Env.new(opts)
 
@@ -73,28 +82,23 @@ class Localite::Template < String
     # get all --> {* code *} <-- parts from the template strings and send
     # them thru the environment.
     gsub(/\{\*([^\}]+?)\*\}/) do |_|
-      Modi.send mode, env[$1]
+      Localite::Format.send format, env[$1]
     end
   end
-
-  module Modi
-    def self.text(s); s; end
-    def self.html(s); CGI.escapeHTML(s); end
-  end
 end
 
 module Localite::Template::Etest
   Template = Localite::Template
   
   def test_templates
-    assert_equal "abc",                   Template.run(:text, "{*xyz*}", :xyz => "abc")
-    assert_equal "3 items",               Template.run(:text, "{*pl 'item', xyz.length*}", :xyz => "abc")
-    assert_equal "xyz",                   Template.run(:text, "xyz", :xyz => "abc")
-    assert_equal "abc",                   Template.run(:text, "{*xyz*}", :xyz => "abc")
-    assert_equal "3",                     Template.run(:text, "{*xyz.length*}", :xyz => "abc")
-    assert_equal "3",                     Template.run(:text, "{*xyz.length*}", :xyz => "abc")
-    assert_equal "3 Fixnums",             Template.run(:text, "{*pl xyz*}", :xyz => [1, 2, 3])
-    assert_equal "3 Fixnums and 1 Float", Template.run(:text, "{*pl xyz*} and {*pl fl*}", :xyz => [1, 2, 3], :fl => [1.0])
+    assert_equal "abc",                   Template.text("{*xyz*}", :xyz => "abc")
+    assert_equal "3 items",               Template.text("{*pl 'item', xyz.length*}", :xyz => "abc")
+    assert_equal "xyz",                   Template.text("xyz", :xyz => "abc")
+    assert_equal "abc",                   Template.text("{*xyz*}", :xyz => "abc")
+    assert_equal "3",                     Template.text("{*xyz.length*}", :xyz => "abc")
+    assert_equal "3",                     Template.text("{*xyz.length*}", :xyz => "abc")
+    assert_equal "3 Fixnums",             Template.text("{*pl xyz*}", :xyz => [1, 2, 3])
+    assert_equal "3 Fixnums and 1 Float", Template.text("{*pl xyz*} and {*pl fl*}", :xyz => [1, 2, 3], :fl => [1.0])
   end
 
   class Name < String
@@ -104,7 +108,7 @@ module Localite::Template::Etest
   end
   
   def test_nohash
-    assert_equal "abc",                   Template.run(:text, "{*name*}", Name.new("abc"))
+    assert_equal "abc",                   Template.text("{*name*}", Name.new("abc"))
   end
 
   def test_pl
@@ -121,20 +125,20 @@ module Localite::Template::Etest
   end
 
   def test_html_env
-    assert_equal "a>c",                 Template.run(:text, "{*xyz*}", :xyz => "a>c")
-    assert_equal "a&gt;c",              Template.run(:html, "{*xyz*}", :xyz => "a>c")
-    assert_equal "> a>c",               Template.run(:text, "> {*xyz*}", :xyz => "a>c")
-    assert_equal "> a&gt;c",            Template.run(:html, "> {*xyz*}", :xyz => "a>c")
+    assert_equal "a>c",                 Template.text("{*xyz*}", :xyz => "a>c")
+    assert_equal "a&gt;c",              Template.html("{*xyz*}", :xyz => "a>c")
+    assert_equal "> a>c",               Template.text("> {*xyz*}", :xyz => "a>c")
+    assert_equal "> a&gt;c",            Template.html("> {*xyz*}", :xyz => "a>c")
   end
 
   def test_template_hash
-    assert_equal "a>c",                 Template.run(:text, "{*xyz*}", :xyz => "a>c")
-    assert_equal "a>c",                 Template.run(:text, "{*xyz*}", "xyz" => "a>c")
+    assert_equal "a>c",                 Template.text("{*xyz*}", :xyz => "a>c")
+    assert_equal "a>c",                 Template.text("{*xyz*}", "xyz" => "a>c")
   end
 
   def test_template_hash_missing
     assert_raise(NameError) {
-      Template.run(:text, "{*abc*}", :xyz => "a>c")
+      Template.text("{*abc*}", :xyz => "a>c")
     }
   end
 end

data/lib/localite/tr.rb

@@ -0,0 +1,292 @@
+class Localite::Backend::Tr 
+  #
+  # This class handles line reading from ios (files, etc.) and strings.
+  # It offers an one-line unread buffer.
+  class IO
+    attr_reader :lineno
+
+    def initialize(src)
+      @lineno = 0
+      @io = if src.is_a?(String)
+        StringIO.new(src)
+      else
+        src
+      end
+    end
+
+    def eof?
+      @unreadline.nil? && @io.eof?
+    end
+
+    def unreadline(line)
+      @unreadline = line
+    end
+
+    def readline
+      r, @unreadline = @unreadline, nil
+      r || begin
+        @lineno += 1
+        @io.readline
+      end
+    end
+  end
+
+  def self.parse(io, name=nil, &block)
+    new(io, name).parse(&block)
+  end
+
+  def self.load(file, &block)
+    new(File.open(file), file).parse(&block)
+  end
+
+  def initialize(src, name=nil)
+    @src, @name = src, name
+  end
+  
+  attr_reader :indent, :name
+
+  def keys
+    return @keys if @keys
+
+    if @parse
+      @keys = @parse.keys
+    else
+      @keys = []
+      parse_ do |k,_|
+        @keys << k
+      end
+      @keys.uniq!
+    end
+
+    @keys.sort!
+  end
+
+  def parse(&block)
+    if block_given?
+      parse_(&block)
+    else
+      @parse ||= begin
+        hash = {}
+        parse_ do |k,v|
+          duplicate_entry(k) if hash.key?(k)
+          hash[k] = v
+        end
+        hash
+      end
+    end
+  end
+
+  protected
+  
+  def duplicate_entry(k)
+    msg = "[#{name}] " if name
+    Localite.logger.warn "#{msg}Duplicate entry" + k.inspect 
+  end
+
+  private
+
+  #
+  # -- TR scopes
+  def register_scope(indent, name)
+    @indent = indent
+    @scopes = @scopes[0, indent]
+    @scopes[indent] = evaluate(name)
+  end
+  
+  def current_scope
+    @scopes[0..@indent].compact.join(".")
+  end
+
+  def parse_(&block)
+    @io = IO.new @src
+    @scopes = []
+    while !io.eof? do
+      line = io.readline
+
+      #
+      # skip empty and comment lines
+      next if line =~ /^\s*(#|$)/
+
+      if line =~ /^(\s*)([^:]+):\s*\|\s*$/                # Start of a multiline entry?
+        register_scope $1.length, $2
+        yield current_scope, read_multiline_value
+      elsif line =~ /^(\s*)([^:]+):\s*([^|].*)/           # A singleline entry?
+        register_scope $1.length, $2
+        value = $3.sub(/\s+$/, "")
+        yield current_scope, evaluate(value) unless value.empty?
+      else
+        msg = name ? "#{name}(#{io.lineno})" : "Line #{io.lineno}"
+
+        msg += ": format error in #{line.inspect}"
+        dlog msg
+        raise msg
+      end
+    end
+  ensure
+    @io = nil
+  end
+
+  attr_reader :io
+  
+  def read_multiline_value
+    value = []
+    while !io.eof? do
+      line = io.readline
+      line =~ /^(\s*)(.*)$/
+      
+      if $2.empty?
+        value << ""
+        next
+      end
+
+      line_indent = $1.length 
+      
+      #
+      # all multiline entries have a higher indent than the current line.
+      if line_indent <= indent
+        io.unreadline(line)
+        break
+      end
+      value << $2.sub(/\s+$/, "")
+    end
+    
+    value.join("\n")
+  end
+
+  #
+  # evaluate a string
+  def evaluate(s)
+    return s unless s[0] == s[-1] && (s[0] == 34 || s[0] == 39)
+
+    s[1..-2].gsub(/\\(.)/m) do
+      case $1
+      when "n"  then "\n"
+      when "t"  then "\t"
+      else      $1
+      end
+    end
+  end
+end
+
+module Localite::Backend::Tr::Etest
+  def evaluate(s)
+    Localite::Backend::Tr.new("").send(:evaluate, s)
+  end
+
+  def test_tr_file
+    d = Localite::Backend::Tr.load(File.dirname(__FILE__) + "/../../test/i18n/en.tr")
+    assert_kind_of(Hash, d)
+  end
+  
+  def test_eval_string_prereq
+    assert_equal 34, '""'[0]
+    assert_equal 34, '""'[-1]
+    assert_equal 39, "''"[0]
+  end
+  
+  def test_eval_string
+    assert_equal "ab\nc", evaluate("'ab\nc'")
+    assert_equal "ab\ncdef", evaluate('"ab\ncdef"')
+    assert_equal "ab\tcd\nef", evaluate('"ab\tcd\nef"')
+    assert_equal "abbcd", evaluate('"ab\bcd"')
+    assert_equal "en/outer/inner/y1", evaluate("\"en/outer/inner/y1\"")
+  end
+  
+  def test_parse_tr
+    tr = <<-TR
+#
+# comment
+param: "{* xxx *}"
+base: "en_only"
+t: "en.t"
+#
+# another comment
+outer:
+  inner:
+    x1: "en/outer/inner/x1"
+  inner:
+    y1: "en/outer/inner/y1"
+title: "This is hypertext"
+title2: "This is <> hypertext"
+
+ml: |
+  mlmlml
+ml: mlober
+
+outer: |
+  A first multiline
+  entry
+outer: Hey Ho!
+outer: |
+  A multiline
+  entry
+TR
+
+    p = Localite::Backend::Tr.new(tr)
+    p.stubs(:dlog) {}
+
+    assert_equal(%w(base ml outer outer.inner.x1 outer.inner.y1 param t title title2), p.keys)
+    data = p.parse
+
+    assert_equal("en/outer/inner/x1", data["outer.inner.x1"])
+    assert_equal(nil, data["outer.inner"])
+    assert_equal("A multiline\nentry", data["outer"])
+    assert_equal("mlober", data["ml"])
+    assert_kind_of(Hash, data)
+  end
+
+  def test_parse_tr_invalid
+    tr = <<-TR
+
+ml: | av
+  mlmlml
+ml: mlober
+TR
+
+    p = Localite::Backend::Tr.new(tr)
+    p.stubs(:dlog) {}
+    assert_raise(RuntimeError) {
+      p.parse
+    }
+  end
+
+  def test_parse_from_io
+    tr = "  ml: mlober"
+    d = Localite::Backend::Tr.parse(tr)
+    assert_equal({"ml" => "mlober"}, d)
+  end
+  
+  def test_parse_key_names
+    tr = <<TR
+  a: aa
+    "b.c": abc
+    "b\\nc": anlc
+    b.c: dot
+TR
+
+    d = Localite::Backend::Tr.parse(tr)
+    assert_equal({"a"=>"aa", "a.b.c"=>"dot", "a.b\nc"=>"anlc"}, d)
+  end
+  
+  def test_multiline_w_spaces
+    tr = <<-TR
+refresh:
+  title:        t1
+  info:           |
+    line1
+    line2
+
+    line3
+    line4
+  title:        t2
+TR
+
+    p = Localite::Backend::Tr.new(tr)
+    d = p.parse
+    p.stubs(:dlog) {}
+ 
+    assert_equal "line1\nline2\n\nline3\nline4", d["refresh.info"]
+    assert_equal "t2", d["refresh.title"]
+  end
+
+end