threedaymonk-l10nizer 0.0.1

data/Rakefile

@@ -0,0 +1,54 @@
+require "rubygems"
+require "rake/gempackagetask"
+require "rake/testtask"
+require "lib/l10nizer/version"
+
+task :default => [:test]
+
+Rake::TestTask.new("test") do |t|
+  t.libs   << "test"
+  t.pattern = "test/**/test_*.rb"
+  t.verbose = true
+end
+
+task :default => :test
+
+require "rake/testtask"
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+spec = Gem::Specification.new do |s|
+  s.name              = "l10nizer"
+  s.version           = L10nizer::VERSION::STRING
+  s.summary           = "Automatically extract strings from ERB templates and replace with calls to t()"
+  s.author            = "Paul Battley"
+  s.email             = "pbattley@gmail.com"
+
+  s.has_rdoc          = false
+
+  s.files             = %w(Rakefile) + Dir.glob("{bin,test,lib}/**/*")
+  s.executables       = FileList["bin/**"].map { |f| File.basename(f) }
+
+  s.require_paths     = ["lib"]
+
+  s.add_dependency("treetop",  "~> 1.2.6")
+  s.add_dependency("polyglot", "~> 0.2.5")
+
+  s.add_development_dependency("shoulda")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+
+  # Generate the gemspec file for github.
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
+desc 'Clear out generated packages'
+task :clean => [:clobber_package] do
+  rm "#{spec.name}.gemspec"
+end

data/bin/l10nizer

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+require "l10nizer"
+require "yaml"
+
+keygen = L10nizer::KeyGenerator.new
+l10ns = {}
+
+templates = Dir["app/views/**/*.html.erb"]
+templates.each do |path|
+  keygen.namespace = path.split("/")[2 .. -2] * "_"
+  source = File.read(path)
+  l10nizer = L10nizer::Processor.new(source, keygen)
+  l10ns.merge!(l10nizer.l10ns)
+  File.open(path, "w") do |f|
+    f << l10nizer.reformed
+  end
+end
+
+File.open("config/locales/l10nized.yml", "w") do |f|
+  f << l10ns.to_yaml
+end

data/lib/l10nizer.rb

@@ -0,0 +1,2 @@
+require "l10nizer/processor"
+require "l10nizer/keygen"

data/lib/l10nizer/grammar.treetop

@@ -0,0 +1,49 @@
+grammar HtmlErb
+  rule document
+    (html_comment / javascript / erb_control / whitespace / just_erb_eval / text / tag)* <Document>
+  end
+
+  rule html_comment
+    "<!--" ([^\-] / "-" [^\-] / "--" [^>])* "-->"
+  end
+
+  rule javascript
+    "<script" ([^<] / "<" [^/] / "</" [^s] / "</s" [^c] / "</sc" [^r] / "</scr" [^i] / "</scri" [^p] / "</scrip" [^t] / "</script" [^>"])* "</script>"
+  end
+
+  rule erb_control
+    "<%" [^=] ruby_code "%>"
+  end
+
+  rule tag
+    "<" (erb_eval / [^>])+ ">"
+  end
+
+  rule whitespace
+    [\s]+
+  end
+
+  rule optional_whitespace
+    [\s]*
+  end
+
+  rule just_erb_eval
+    (optional_whitespace erb_eval)+
+  end
+
+  rule text
+    (optional_whitespace erb_eval / optional_whitespace word)+ <Text>
+  end
+
+  rule word
+    [^<\s]+
+  end
+
+  rule erb_eval
+    "<%=" ruby_code "%>" <Eval>
+  end
+
+  rule ruby_code
+    (("%" [^>]) / [^%])+
+  end
+end

data/lib/l10nizer/keygen.rb

@@ -0,0 +1,33 @@
+module L10nizer
+  class KeyGenerator
+    attr_accessor :namespace
+
+    def initialize
+      @seen = {}
+    end
+
+    def call(string)
+      provisional = [make_safe(namespace), make_safe(string)].compact * "."
+
+      until try(provisional, string)
+        provisional.sub!(/(?:_\d+)?$/){ |m| "_" + m.to_i.succ.to_s }
+      end
+
+      return provisional
+    end
+
+    def try(key, string)
+      if [nil, string].include?(@seen[key])
+        @seen[key] = string
+        true
+      else
+        false
+      end
+    end
+
+    def make_safe(string)
+      return nil if string.nil?
+      string.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_|_$/, "")[0, 40]
+    end
+  end
+end

data/lib/l10nizer/node.rb

@@ -0,0 +1,96 @@
+module L10nizer
+  class NodeWrapperFactory
+    def self.wrap(node, keygen=nil)
+      case node
+      when HtmlErb::Text
+        TextNode.new(node, keygen)
+      when HtmlErb::Eval
+        EvalNode.new(node, keygen)
+      else
+        BasicNode.new(node)
+      end
+    end
+  end
+
+  class BasicNode
+    def initialize(node, keygen=nil)
+      @node   = node
+      @keygen = keygen
+    end
+
+    def l10n
+      {}
+    end
+
+    def to_s
+      @node.text_value
+    end
+
+    def evaluated?
+      false
+    end
+  end
+
+  class TextNode < BasicNode
+    def l10n
+      _, text = vars_and_text
+      text ? {key => text} : {}
+    end
+
+    def to_s
+      vars, _ = vars_and_text
+      return super unless vars
+
+      params = ['"' + key + '"']
+      vars.each_with_index do |v, i|
+        params << %{:#{variable_name(i)} => #{v}}
+      end
+
+      %{<%= t(#{params * ", "}) %>}
+    end
+
+  private
+    def children
+      @node.children.map{ |e| NodeWrapperFactory.wrap(e) }
+    end
+
+    def variable_name(index)
+      ("a" .. "z").to_a[index]
+    end
+
+    def vars_and_text
+      @vars_and_text ||= (
+        if children.all?{ |c| c.evaluated? }
+          []
+        else
+          l10n = ""
+          vars = []
+          children.each do |e|
+            if e.evaluated?
+              l10n << "{{#{variable_name(vars.length)}}}"
+              vars << e.to_s
+            else
+              l10n << e.to_s
+            end
+          end
+          [vars, l10n]
+        end
+      )
+    end
+
+    def key
+      _, text = vars_and_text
+      @key ||= @keygen.call(text)
+    end
+  end
+
+  class EvalNode < BasicNode
+    def to_s
+      super[/\A<%=\s*(.*?)\s*%>\Z/, 1]
+    end
+
+    def evaluated?
+      true
+    end
+  end
+end

data/lib/l10nizer/parser.rb

@@ -0,0 +1,17 @@
+require "treetop"
+require "polyglot"
+require "l10nizer/grammar"
+
+module HtmlErb
+  class Document < Treetop::Runtime::SyntaxNode
+  end
+
+  class Text < Treetop::Runtime::SyntaxNode
+    def children
+      elements.map{ |e| e.elements }.flatten
+    end
+  end
+
+  class Eval < Treetop::Runtime::SyntaxNode
+  end
+end

data/lib/l10nizer/processor.rb

@@ -0,0 +1,23 @@
+require "l10nizer/parser"
+require "l10nizer/node"
+
+module L10nizer
+  class Processor
+    def initialize(html, keygen)
+      @html = html
+      @keygen = keygen
+    end
+
+    def l10ns
+      processed.inject({}){ |hash, node| hash.merge(node.l10n) }
+    end
+
+    def reformed
+      processed.map{ |e| e.to_s }.join
+    end
+
+    def processed
+      @processed ||= HtmlErbParser.new.parse(@html).elements.map{ |e| NodeWrapperFactory.wrap(e, @keygen) }
+    end
+  end
+end

data/lib/l10nizer/version.rb

@@ -0,0 +1,9 @@
+module L10nizer #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/test/samples/input.html.erb

@@ -0,0 +1,34 @@
+<li id="action_<%=h thing.whatsit.id %>" class="item underway">
+  <div class="wrap">
+    <div class="placeholder">
+      <%- if thing.whatsit.identity? -%>
+        <%- if thing.whatsit.identity.processed? -%>
+          <%= image_tag thing.whatsit.identity.url(:medium) %>
+        <%- else -%>
+          <%= t("general.processing_image") %>
+        <%- end -%>
+      <%- end -%>
+    </div>
+    <h3>
+      <% link_to(thing) do %>
+        <span><%=h thing.whatsit.name %></span><%=h thing.short_address %>
+      <% end %>
+    </h3>
+  </div>
+  <div class="details">
+    <div class="format">
+      <h4 class="obfuscate">Skills</h4>
+      <ul class="skills">
+        <% thing.whatsit.skills.each do |skill| %>
+          <li class="skill">
+            <%= skill_icon(skill, :small) %>
+          </li>
+        <% end %>
+        <li class="engagement">
+          <%= engagement_icon(4, :small) %>
+        </li>
+      </ul>
+      <%=h truncate(thing.whatsit.short_description, :length => 130) %>
+    </div>
+  </div>
+</li>

data/test/samples/output.html.erb

@@ -0,0 +1,34 @@
+<li id="action_<%=h thing.whatsit.id %>" class="item underway">
+  <div class="wrap">
+    <div class="placeholder">
+      <%- if thing.whatsit.identity? -%>
+        <%- if thing.whatsit.identity.processed? -%>
+          <%= image_tag thing.whatsit.identity.url(:medium) %>
+        <%- else -%>
+          <%= t("general.processing_image") %>
+        <%- end -%>
+      <%- end -%>
+    </div>
+    <h3>
+      <% link_to(thing) do %>
+        <span><%=h thing.whatsit.name %></span><%=h thing.short_address %>
+      <% end %>
+    </h3>
+  </div>
+  <div class="details">
+    <div class="format">
+      <h4 class="obfuscate"><%= t("skills") %></h4>
+      <ul class="skills">
+        <% thing.whatsit.skills.each do |skill| %>
+          <li class="skill">
+            <%= skill_icon(skill, :small) %>
+          </li>
+        <% end %>
+        <li class="engagement">
+          <%= engagement_icon(4, :small) %>
+        </li>
+      </ul>
+      <%=h truncate(thing.whatsit.short_description, :length => 130) %>
+    </div>
+  </div>
+</li>

data/test/test_key_generator.rb

@@ -0,0 +1,56 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "test/unit"
+require "l10nizer/keygen"
+require "shoulda"
+
+class KeyGeneratorTest < Test::Unit::TestCase
+
+  def setup
+    @keygen = L10nizer::KeyGenerator.new
+  end
+
+  context "without namespacing" do
+    should "generate keys based on string" do
+      assert_equal "foo_bar_baz_a", @keygen.call("Foo bar  baz  {{a}}")
+    end
+
+    should "truncate exceptionally long keys" do
+      long  = "blah_" * 20
+      short = long[0, 40]
+      assert_equal short, @keygen.call(long)
+    end
+
+    should "reuse key for same text" do
+      assert_equal @keygen.call("the same"), @keygen.call("the same")
+    end
+
+    should "prevent duplicate keys for different texts" do
+      assert_equal "a_thing",   @keygen.call("a thing")
+      assert_equal "a_thing_1", @keygen.call("A thing")
+      assert_equal "a_thing_2", @keygen.call("A Thing")
+    end
+  end
+
+  context "with namespacing" do
+    setup do
+      @keygen.namespace = "ns1"
+    end
+
+    should "prepend namespace" do
+      assert_equal "ns1.foo", @keygen.call("Foo")
+    end
+
+    should "prevent duplicate keys for different texts" do
+      assert_equal "ns1.a_thing",   @keygen.call("a thing")
+      assert_equal "ns1.a_thing_1", @keygen.call("A thing")
+      assert_equal "ns1.a_thing_2", @keygen.call("A Thing")
+    end
+
+    should "check duplication by namespace" do
+      assert_equal "ns1.a_thing", @keygen.call("a thing")
+      @keygen.namespace = "ns2"
+      assert_equal "ns2.a_thing", @keygen.call("A thing")
+    end
+  end
+end

data/test/test_processor.rb

@@ -0,0 +1,105 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "test/unit"
+require "l10nizer/processor"
+require "shoulda"
+
+class ProcessorTest < Test::Unit::TestCase
+
+  class DumbKeyGenerator
+    def call(string)
+      string.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_|_$/, "")[0, 40]
+    end
+  end
+
+  context "when finding text" do
+    setup do
+      html = "just some text"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass key to t()" do
+      expected = %{<%= t("just_some_text") %>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings" do
+      expected = {"just_some_text" => "just some text"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  context "when interpolating inline eval" do
+    setup do
+      html = "String <%= 27 %> with <%= 42 %>"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass values to t()" do
+      expected = %{<%= t("string_a_with_b", :a => 27, :b => 42) %>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings" do
+      expected = {"string_a_with_b" => "String {{a}} with {{b}}"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  context "when interpolating in multiple text strings" do
+    setup do
+      html = "<p>String <%= 27 %> with <%= 42 %></p><p>Another <%= 'x' %></p>"
+      @l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    end
+
+    should "pass values to t() reusing placeholder variables" do
+      expected = %{<p><%= t("string_a_with_b", :a => 27, :b => 42) %></p><p><%= t("another_a", :a => 'x') %></p>}
+      assert_equal expected, @l10nizer.reformed
+    end
+
+    should "extract l10n strings with placeholder variables" do
+      expected = {"string_a_with_b" => "String {{a}} with {{b}}", "another_a" => "Another {{a}}"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+  should "not try to localise inline eval on its own" do
+    html = "<p><%= 27 %></p>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise an HTML comment" do
+    html = "<!-- <p><%= 27 %></p> --> <p> <!-- fooo --> </p>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  should "not try to localise Javascript" do
+    html = "<script>var a = 3;</script> <script>var b = 'b';</script>"
+    l10nizer = L10nizer::Processor.new(html, DumbKeyGenerator.new)
+    assert_equal html, l10nizer.reformed
+  end
+
+  context "when parsing a sample document" do
+    setup do
+      @html     = File.read(File.join(File.dirname(__FILE__), "samples", "input.html.erb"))
+      @expected = File.read(File.join(File.dirname(__FILE__), "samples", "output.html.erb"))
+      @l10nizer = L10nizer::Processor.new(@html, DumbKeyGenerator.new)
+    end
+
+    should "replace embedded text" do
+      actual = @l10nizer.reformed
+      #@expected.split(/\n/).zip(actual.split(/\n/)).each do |a, b|
+      #  puts a, b unless a == b
+      #end
+      assert_equal @expected, actual
+    end
+
+    should "extract l10n strings" do
+      expected = {"skills" => "Skills"}
+      assert_equal expected, @l10nizer.l10ns
+    end
+  end
+
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: threedaymonk-l10nizer
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Paul Battley
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-10 00:00:00 -07:00
+default_executable: l10nizer
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: treetop
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.2.6
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: polyglot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.2.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: pbattley@gmail.com
+executables: 
+- l10nizer
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- bin/l10nizer
+- test/samples
+- test/samples/input.html.erb
+- test/samples/output.html.erb
+- test/test_key_generator.rb
+- test/test_processor.rb
+- lib/l10nizer
+- lib/l10nizer/grammar.treetop
+- lib/l10nizer/keygen.rb
+- lib/l10nizer/node.rb
+- lib/l10nizer/parser.rb
+- lib/l10nizer/processor.rb
+- lib/l10nizer/version.rb
+- lib/l10nizer.rb
+has_rdoc: false
+homepage: 
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 3
+summary: Automatically extract strings from ERB templates and replace with calls to t()
+test_files: []
+