l10nizer 0.0.9

data/README.md

@@ -0,0 +1,50 @@
+L10nizer
+========
+
+Automagic _ex post facto_ localisation for Rails templates.
+
+What it does
+------------
+
+Processes all your `html.erb` templates, extracts text, replaces it with `t()` calls, and generates a YAML file of localisations.
+
+For example, given a file `app/views/things/show.html.erb` with this content:
+
+    <div class="thing">
+      <h1>Some heading</h1>
+      <p>This thing is called <%= h(@thing.name)</p>
+    </div>
+
+l10nizer will change it to:
+
+    <div class="thing">
+      <h1><%= t("things.some_heading") %></h1>
+      <p><%= t("things.this_thing_is_called_a", :a => (h(@thing.name))) %></p>
+    </div>
+
+and generate the following entries in `config/locales/l10nized.yml`:
+
+    things:
+      some_heading: Some heading
+      this_thing_is_called_a: This thing is called {{a}}
+
+You can then use `l10nized.yml` as a basis for the localisation file for your current locale, e.g. `en_GB.yml`.
+
+Usage
+-----
+
+From within a Rails application directory:
+
+    l10nizer
+
+Specifying the application path explicitly:
+
+    l10nizer /path/to/my/rails/app
+
+Limitations
+-----------
+
+* Perhaps ironically for a _localisation_ utility, l10nizer assumes that your templates are written in English or generally in ASCII, and ignores non-alphanumeric content when generating localisation keys. This could be fixed by modifying or replacing the L10nizer::KeyGenerator class.
+* L10nizer takes no position on HTML entities or escaping. You __will__ need to review the changes it makes.
+* Similarly, pluralisation is outside the scope of this application and will require attention.
+* Strings that should be single entities but which contain HTML will be broken into multiple localisation strings.

data/Rakefile

@@ -0,0 +1,54 @@
+require "rubygems"
+require "rake/gempackagetask"
+require "rake/testtask"
+require "lib/l10nizer/version"
+
+task :default => [:test]
+
+Rake::TestTask.new("test") do |t|
+  t.libs   << "test"
+  t.pattern = "test/**/test_*.rb"
+  t.verbose = true
+end
+
+task :default => :test
+
+require "rake/testtask"
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+spec = Gem::Specification.new do |s|
+  s.name              = "l10nizer"
+  s.version           = L10nizer::VERSION::STRING
+  s.summary           = "Automatically extract strings from ERB templates and replace with calls to t()"
+  s.author            = "Paul Battley"
+  s.email             = "pbattley@gmail.com"
+
+  s.has_rdoc          = false
+
+  s.files             = %w(Rakefile README.md) + Dir.glob("{bin,test,lib}/**/*")
+  s.executables       = FileList["bin/**"].map { |f| File.basename(f) }
+
+  s.require_paths     = ["lib"]
+
+  s.add_dependency("treetop",  ">= 1.2.6")
+  s.add_dependency("polyglot", ">= 0.2.5")
+
+  s.add_development_dependency("thoughtbot-shoulda")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+
+  # Generate the gemspec file for github.
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
+desc 'Clear out generated packages'
+task :clean => [:clobber_package] do
+  rm "#{spec.name}.gemspec"
+end

data/bin/l10nizer

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+require "l10nizer"
+require "yaml"
+require "fileutils"
+
+Dir.chdir(ARGV.first) if ARGV.first
+templates = Dir["app/views/**/*.html.erb"]
+raise "Can't find any templates in app/views." unless templates.any?
+
+keygen = L10nizer::KeyGenerator.new
+l10ns = {}
+
+templates.each do |path|
+  keygen.namespace = path.split("/")[2]
+  source = File.read(path)
+  l10nizer = L10nizer::Processor.new(source, keygen)
+  l10ns.merge!(l10nizer.l10ns)
+  File.open(path, "w") do |f|
+    f << l10nizer.reformed
+  end
+end
+
+l10ns = l10ns.inject({}){ |hash, (key, value)|
+  parts = key.split(".")
+  parts[0 .. -2].inject(hash){ |h, k| h[k] ||= {} }[parts.last] = value
+  hash
+}
+
+FileUtils.mkdir_p("config/locales")
+File.open("config/locales/l10nized.yml", "w") do |f|
+  f << l10ns.to_yaml
+end

data/lib/l10nizer/grammar.treetop

@@ -0,0 +1,53 @@
+grammar HtmlErb
+  rule document
+    (html_comment / javascript / style / erb_control / whitespace / text / tag)* <Document>
+  end
+
+  rule html_comment
+    "<!--" ([^\-] / "-" [^\-] / "--" [^>])* "-->"
+  end
+
+  rule javascript
+    "<script" ([^<] / "<" [^/] / "</" [^s] / "</s" [^c] / "</sc" [^r] / "</scr" [^i] / "</scri" [^p] / "</scrip" [^t] / "</script" [^>"])* "</script>"
+  end
+
+  rule style
+    "<style" ([^<] / "<" [^/] / "</" [^s] / "</s" [^t] / "</st" [^y] / "</sty" [^l] / "</styl" [^e] / "</style" [^>"])* "</style>"
+  end
+
+  rule erb_control
+    "<%" [^=] ruby_code "%>"
+  end
+
+  rule tag
+    "<" (erb_control / erb_eval / [^>])+ ">"
+  end
+
+  rule whitespace
+    [\s]+
+  end
+
+  rule optional_whitespace
+    [\s]*
+  end
+
+  rule text
+    (optional_whitespace (erb_eval / inline_markup / word))+ <Text>
+  end
+
+  rule inline_markup
+    "<" "/"? ("em" / "strong") [^>]* ">"
+  end
+
+  rule word
+    [^<\s]+ <Word>
+  end
+
+  rule erb_eval
+    "<%=" ruby_code "%>" <Eval>
+  end
+
+  rule ruby_code
+    (("%" [^>]) / [^%])+
+  end
+end

data/lib/l10nizer/keygen.rb

@@ -0,0 +1,41 @@
+module L10nizer
+  class KeyGenerator
+    attr_accessor :namespace
+
+    def initialize
+      @seen = {}
+    end
+
+    def call(string)
+      provisional = [make_safe(namespace), make_safe(string)].compact * "."
+
+      until try(provisional, string)
+        provisional.sub!(/(?:_\d+)?$/){ |m| "_" + m.to_i.succ.to_s }
+      end
+
+      return provisional
+    end
+
+    def try(key, string)
+      if [nil, string].include?(@seen[key])
+        @seen[key] = string
+        true
+      else
+        false
+      end
+    end
+
+    def make_safe(string)
+      return nil if string.nil?
+      safe = string.
+             downcase.
+             gsub(/&[a-z0-9]{1,20};/, ""). # entities
+             gsub(/<[^>]*>/, "").          # html
+             gsub(/[^a-z0-9]+/, "_").      # non alphanumeric
+             slice(0, 40).
+             gsub(/^_|_$/, "")             # leading/trailing _
+      safe = "unknown" if safe.empty?
+      safe
+    end
+  end
+end

data/lib/l10nizer/node.rb

@@ -0,0 +1,108 @@
+module L10nizer
+  class NodeWrapperFactory
+    def self.wrap(node, keygen=nil)
+      case node
+      when HtmlErb::Text
+        TextNode.new(node, keygen)
+      when HtmlErb::Eval
+        EvalNode.new(node, keygen)
+      when HtmlErb::Word
+        WordNode.new(node)
+      else
+        BasicNode.new(node)
+      end
+    end
+  end
+
+  class BasicNode
+    def initialize(node, keygen=nil)
+      @node   = node
+      @keygen = keygen
+    end
+
+    def l10n
+      {}
+    end
+
+    def to_s
+      @node.text_value
+    end
+
+    def evaluated?
+      false
+    end
+
+    def string?
+      false
+    end
+  end
+
+  class TextNode < BasicNode
+    def l10n
+      _, text = vars_and_text
+      text ? {key => text} : {}
+    end
+
+    def to_s
+      vars, _ = vars_and_text
+      return super unless vars
+
+      params = ['"' + key + '"']
+      vars.each_with_index do |v, i|
+        params << %{:#{variable_name(i)} => (#{v})}
+      end
+
+      %{<%= t(#{params * ", "}) %>}
+    end
+
+  private
+    def children
+      @node.children.map{ |e| NodeWrapperFactory.wrap(e) }
+    end
+
+    def variable_name(index)
+      ("a" .. "z").to_a[index]
+    end
+
+    def vars_and_text
+      @vars_and_text ||= (
+        if children.all?{ |c| c.evaluated? } || !children.any?{ |c| c.string? }
+          []
+        else
+          l10n = ""
+          vars = []
+          children.each do |e|
+            if e.evaluated?
+              l10n << "{{#{variable_name(vars.length)}}}"
+              vars << e.to_s
+            else
+              l10n << e.to_s
+            end
+          end
+          [vars, l10n]
+        end
+      )
+    end
+
+    def key
+      _, text = vars_and_text
+      @key ||= @keygen.call(text)
+    end
+  end
+
+  class EvalNode < BasicNode
+    def to_s
+      super[/\A<%=\s*(.*?)\s*%>\Z/, 1]
+    end
+
+    def evaluated?
+      true
+    end
+  end
+
+  class WordNode < BasicNode
+    def string?
+      true
+    end
+  end
+end

data/lib/l10nizer/parser.rb

@@ -0,0 +1,20 @@
+require "treetop"
+require "polyglot"
+require "l10nizer/grammar"
+
+module HtmlErb
+  class Document < Treetop::Runtime::SyntaxNode
+  end
+
+  class Text < Treetop::Runtime::SyntaxNode
+    def children
+      elements.map{ |e| e.elements }.flatten
+    end
+  end
+
+  class Eval < Treetop::Runtime::SyntaxNode
+  end
+
+  class Word < Treetop::Runtime::SyntaxNode
+  end
+end

data/lib/l10nizer/processor.rb

@@ -0,0 +1,29 @@
+require "l10nizer/parser"
+require "l10nizer/node"
+
+module L10nizer
+  class Processor
+    def initialize(html, keygen)
+      @html = html
+      @keygen = keygen
+    end
+
+    def l10ns
+      processed.inject({}){ |hash, node| hash.merge(node.l10n) }
+    end
+
+    def reformed
+      processed.map{ |e| e.to_s }.join
+    end
+
+    def processed
+      unless @processed
+        kcode = $KCODE
+        $KCODE = "n"
+        @processed = HtmlErbParser.new.parse(@html).elements.map{ |e| NodeWrapperFactory.wrap(e, @keygen) }
+        $KCODE = kcode
+      end
+      @processed
+    end
+  end
+end

data/lib/l10nizer/version.rb

@@ -0,0 +1,9 @@
+module L10nizer #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 9
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/l10nizer.rb

@@ -0,0 +1,2 @@
+require "l10nizer/processor"
+require "l10nizer/keygen"

data/test/samples/input.html.erb

@@ -0,0 +1,34 @@
+<li id="action_<%=h thing.whatsit.id %>" class="item underway">
+  <div class="wrap">
+    <div class="placeholder">
+      <%- if thing.whatsit.identity? -%>
+        <%- if thing.whatsit.identity.processed? -%>
+          <%= image_tag thing.whatsit.identity.url(:medium) %>
+        <%- else -%>
+          <%= t("general.processing_image") %>
+        <%- end -%>
+      <%- end -%>
+    </div>
+    <h3>
+      <% link_to(thing) do %>
+        <span><%=h thing.whatsit.name %></span><%=h thing.short_address %>
+      <% end %>
+    </h3>
+  </div>
+  <div class="details">
+    <div class="format">
+      <h4 class="obfuscate">Skills</h4>
+      <ul class="skills">
+        <% thing.whatsit.skills.each do |skill| %>
+          <li class="skill">
+            <%= skill_icon(skill, :small) %>
+          </li>
+        <% end %>
+        <li class="engagement">
+          <%= engagement_icon(4, :small) %>
+        </li>
+      </ul>
+      <%=h truncate(thing.whatsit.short_description, :length => 130) %>
+    </div>
+  </div>
+</li>

data/test/samples/output.html.erb

@@ -0,0 +1,34 @@
+<li id="action_<%=h thing.whatsit.id %>" class="item underway">
+  <div class="wrap">
+    <div class="placeholder">
+      <%- if thing.whatsit.identity? -%>
+        <%- if thing.whatsit.identity.processed? -%>
+          <%= image_tag thing.whatsit.identity.url(:medium) %>
+        <%- else -%>
+          <%= t("general.processing_image") %>
+        <%- end -%>
+      <%- end -%>
+    </div>
+    <h3>
+      <% link_to(thing) do %>
+        <span><%=h thing.whatsit.name %></span><%=h thing.short_address %>
+      <% end %>
+    </h3>
+  </div>
+  <div class="details">
+    <div class="format">
+      <h4 class="obfuscate"><%= t("skills") %></h4>
+      <ul class="skills">
+        <% thing.whatsit.skills.each do |skill| %>
+          <li class="skill">
+            <%= skill_icon(skill, :small) %>
+          </li>
+        <% end %>
+        <li class="engagement">
+          <%= engagement_icon(4, :small) %>
+        </li>
+      </ul>
+      <%=h truncate(thing.whatsit.short_description, :length => 130) %>
+    </div>
+  </div>
+</li>

data/test/test_key_generator.rb

@@ -0,0 +1,68 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "test/unit"
+require "l10nizer/keygen"
+require "shoulda"
+
+class KeyGeneratorTest < Test::Unit::TestCase
+
+  def setup
+    @keygen = L10nizer::KeyGenerator.new
+  end
+
+  context "without namespacing" do
+    should "generate keys based on string" do
+      assert_equal "foo_bar_baz_a", @keygen.call("Foo bar  baz  {{a}}")
+    end
+
+    should "truncate exceptionally long keys" do
+      long  = "blah_" * 20
+      short = "blah_blah_blah_blah_blah_blah_blah_blah"
+      assert_equal short, @keygen.call(long)
+    end
+
+    should "reuse key for same text" do
+      assert_equal @keygen.call("the same"), @keygen.call("the same")
+    end
+
+    should "prevent duplicate keys for different texts" do
+      assert_equal "a_thing",   @keygen.call("a thing")
+      assert_equal "a_thing_1", @keygen.call("A thing")
+      assert_equal "a_thing_2", @keygen.call("A Thing")
+    end
+
+    should "generate non empty keys for punctuation" do
+      assert_not_equal "", @keygen.call("<>!@#%#.,")
+    end
+
+    should "skip entities in keys" do
+      assert_equal "foo_bar", @keygen.call("foo &apos; bar")
+    end
+
+    should "skip inline markup in keys" do
+      assert_equal "foo_bar", @keygen.call("foo <strong>bar</strong>")
+    end
+  end
+
+  context "with namespacing" do
+    setup do
+      @keygen.namespace = "ns1"
+    end
+
+    should "prepend namespace" do
+      assert_equal "ns1.foo", @keygen.call("Foo")
+    end
+
+    should "prevent duplicate keys for different texts" do
+      assert_equal "ns1.a_thing",   @keygen.call("a thing")
+      assert_equal "ns1.a_thing_1", @keygen.call("A thing")
+      assert_equal "ns1.a_thing_2", @keygen.call("A Thing")
+    end
+
+    should "check duplication by namespace" do
+      assert_equal "ns1.a_thing", @keygen.call("a thing")
+      @keygen.namespace = "ns2"
+      assert_equal "ns2.a_thing", @keygen.call("A thing")
+    end
+  end
+end

data/test/test_processor.rb

@@ -0,0 +1,162 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "test/unit"
+require "l10nizer/processor"
+require "shoulda"
+
+class ProcessorTest < Test::Unit::TestCase
+
+  class DumbKeyGenerator
+    def call(string)
+      string.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_|_$/, "")[0, 40]
+    end
+  end
+
+  context "when finding text" do
+    setup do
+      html = "just some text"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass key to t()" do
+      expected = %{<%= t("just_some_text") %>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings" do
+      expected = {"just_some_text" => "just some text"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  context "when interpolating inline eval" do
+    setup do
+      html = "String <%= 27 %> with <%= 42 %>"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass values to t()" do
+      expected = %{<%= t("string_a_with_b", :a => (27), :b => (42)) %>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings" do
+      expected = {"string_a_with_b" => "String {{a}} with {{b}}"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  context "when interpolating in multiple text strings" do
+    setup do
+      html = "<p>String <%= 27 %> with <%= 42 %></p><p>Another <%= 'x' %></p>"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass values to t() reusing placeholder variables" do
+      expected = %{<p><%= t("string_a_with_b", :a => (27), :b => (42)) %></p><p><%= t("another_a", :a => ('x')) %></p>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings with placeholder variables" do
+      expected = {"string_a_with_b" => "String {{a}} with {{b}}", "another_a" => "Another {{a}}"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  should "not try to localise inline eval on its own" do
+    html = "<p><%= 27 %></p>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise an HTML comment" do
+    html = "<!-- <p><%= 27 %></p> --> <p> <!-- fooo --> </p>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise Javascript" do
+    html = "<script>var a = 3;</script> <script>var b = 'b';</script>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise inline styles" do
+    html = %{<style type="text/css">\nhtml.js .nojs {display: none; background:#fff!important;}\n</style>}
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise control inside a tag" do
+    html = %{<div class="user-skills block <% unless @user.skills.any? %>blank<% end %>">}
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  context "when string contains inline markup" do
+    setup do
+      html = "<p>String with <strong>strong</strong> and <em>emphasised</em> text</p>"
+      @l10nizer = L10nizer::Processor.new(html, lambda{ "key" })
+    end
+
+    should "include that markup in text" do
+      expected = "String with <strong>strong</strong> and <em>emphasised</em> text"
+      assert_equal [expected], @l10nizer.l10ns.values
+    end
+
+    should "use only one localisation" do
+      expected = %{<p><%= t("key") %></p>}
+      assert_equal expected, @l10nizer.reformed
+    end
+  end
+
+  should "not consider <span> to be inline markup" do
+    html = %{foo <span>bar</span>}
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal ["bar", "foo"], l10nizer.l10ns.values.sort
+  end
+
+  context "when parsing a sample document" do
+    setup do
+      @html     = File.read(File.join(File.dirname(__FILE__), "samples", "input.html.erb"))
+      @expected = File.read(File.join(File.dirname(__FILE__), "samples", "output.html.erb"))
+      @l10nizer = L10nizer::Processor.new(@html, DumbKeyGenerator.new)
+    end
+
+    should "replace embedded text" do
+      actual = @l10nizer.reformed
+      #@expected.split(/\n/).zip(actual.split(/\n/)).each do |a, b|
+      #  puts a, b unless a == b
+      #end
+      assert_equal @expected, actual
+    end
+
+    should "extract l10n strings" do
+      expected = {"skills" => "Skills"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  context "when $KCODE is 'UTF8'" do
+    setup do
+      @kcode = $KCODE
+      $KCODE = "UTF8"
+    end
+
+    teardown do
+      $KCODE = @kcode
+    end
+
+    should "parse multi-byte characters in strings" do
+      html = "<p>We’ve</p>"
+      l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+      assert_equal ["We’ve"], l10nizer.l10ns.values
+    end
+
+    should "not change $KCODE" do
+      l10nizer = L10nizer::Processor.new("", DumbKeyGenerator.new)
+      ignore = l10nizer.l10ns.values
+      assert_equal "UTF8", $KCODE
+    end
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: l10nizer
+version: !ruby/object:Gem::Version 
+  version: 0.0.9
+platform: ruby
+authors: 
+- Paul Battley
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-11 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: treetop
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.6
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: polyglot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.2.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: pbattley@gmail.com
+executables: 
+- l10nizer
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- README.md
+- bin/l10nizer
+- test/samples/output.html.erb
+- test/samples/input.html.erb
+- test/test_key_generator.rb
+- test/test_processor.rb
+- lib/l10nizer/grammar.treetop
+- lib/l10nizer/node.rb
+- lib/l10nizer/parser.rb
+- lib/l10nizer/processor.rb
+- lib/l10nizer/keygen.rb
+- lib/l10nizer/version.rb
+- lib/l10nizer.rb
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Automatically extract strings from ERB templates and replace with calls to t()
+test_files: []
+