abroad 1.0.0

data/lib/abroad/extractors/yaml/yaml_extractor.rb

@@ -0,0 +1,28 @@
+require 'yaml'
+
+module Abroad
+  module Extractors
+    module Yaml
+
+      class YamlExtractor < Extractor
+        private
+
+        def parse
+          YAML.load(clean_yaml(stream.read))
+        rescue Psych::SyntaxError => e
+          raise Abroad::SyntaxError, "Syntax error in yaml: #{e.message}"
+        end
+
+        def clean_yaml(yaml_content)
+          if Abroad.jruby?
+            require 'abroad/extractors/yaml/jruby_compat'
+            JRubyCompat.clean(yaml_content)
+          else
+            yaml_content
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/extractors/yaml.rb

@@ -0,0 +1,9 @@
+module Abroad
+  module Extractors
+    module Yaml
+      autoload :DottedKeyExtractor, 'abroad/extractors/yaml/dotted_key_extractor'
+      autoload :RailsExtractor,     'abroad/extractors/yaml/rails_extractor'
+      autoload :YamlExtractor,      'abroad/extractors/yaml/yaml_extractor'
+    end
+  end
+end

data/lib/abroad/extractors.rb

@@ -0,0 +1,28 @@
+module Abroad
+  module Extractors
+    autoload :Extractor, 'abroad/extractors/extractor'
+    autoload :Json,      'abroad/extractors/json'
+    autoload :Xml,       'abroad/extractors/xml'
+    autoload :Yaml,      'abroad/extractors/yaml'
+
+    class << self
+      def register(id, klass)
+        registered[id.to_s] = klass
+      end
+
+      def get(id)
+        registered[id.to_s]
+      end
+
+      def available
+        registered.keys
+      end
+
+      private
+
+      def registered
+        @registered ||= {}
+      end
+    end
+  end
+end

data/lib/abroad/serializers/json/json_serializer.rb

@@ -0,0 +1,27 @@
+require 'json-write-stream'
+
+module Abroad
+  module Serializers
+    module Json
+
+      class JsonSerializer < Serializer
+        attr_reader :writer
+
+        def initialize(stream, locale, encoding = Encoding::UTF_8)
+          super
+          @writer = JsonWriteStream.from_stream(stream, encoding)
+        end
+
+        def write_raw(text)
+          writer.stream.write(text)
+        end
+
+        def flush
+          writer.flush
+          stream.flush
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/json/key_value_serializer.rb

@@ -0,0 +1,20 @@
+module Abroad
+  module Serializers
+    module Json
+
+      class KeyValueSerializer < JsonSerializer
+        def initialize(stream, locale, encoding = Encoding::UTF_8)
+          super
+          writer.write_object
+        end
+
+        def write_key_value(key, value)
+          writer.write_key_value(
+            key.encode(encoding), value.encode(encoding)
+          )
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/json.rb

@@ -0,0 +1,8 @@
+module Abroad
+  module Serializers
+    module Json
+      autoload :JsonSerializer, 'abroad/serializers/json/json_serializer'
+      autoload :KeyValueSerializer, 'abroad/serializers/json/key_value_serializer'
+    end
+  end
+end

data/lib/abroad/serializers/serializer.rb

@@ -0,0 +1,52 @@
+module Abroad
+  module Serializers
+
+    class Serializer
+      attr_reader :stream, :locale, :encoding
+
+      class << self
+        def from_stream(stream, locale)
+          serializer = new(stream, locale)
+
+          if block_given?
+            yield(serializer).tap do
+              serializer.close
+            end
+          else
+            serializer
+          end
+        end
+
+        def open(file, locale, &block)
+          from_stream(File.open(file), locale, &block)
+        end
+      end
+
+      def initialize(stream, locale, encoding = Encoding::UTF_8)
+        @stream = stream
+        @locale = locale
+        @encoding = encoding
+      end
+
+      def write_key_value(key, value)
+        raise NotImplementedError,
+          'expected to be implemented in derived classes'
+      end
+
+      def write_raw(text)
+        raise NotImplementedError,
+          'expected to be implemented in derived classes'
+      end
+
+      def flush
+        stream.flush
+      end
+
+      def close
+        flush
+        stream.close
+      end
+    end
+
+  end
+end

data/lib/abroad/serializers/trie.rb

@@ -0,0 +1,76 @@
+module Abroad
+  module Serializers
+
+    class Trie
+      attr_reader :root
+
+      def initialize(root = nil)
+        @root = root || TrieNode.new
+      end
+
+      def add(key_enum, value)
+        node = root
+
+        key_enum.each do |key|
+          if node.has_child?(key)
+            node = node.child_at(key)
+          else
+            node = node.add_child(key, TrieNode.new)
+          end
+        end
+
+        node.value = value
+      end
+
+      def find(key_enum)
+        node = root
+        key_enum.each do |key|
+          node = node.child_at(key)
+          return nil unless node
+        end
+        node.value
+      end
+    end
+
+    class TrieNode
+      NO_VALUE = :__novalue__
+
+      attr_reader :children
+      attr_accessor :value
+
+      def initialize(value = NO_VALUE)
+        @value = value
+        @children = {}
+      end
+
+      def each_child
+        if block_given?
+          children.each_pair { |key, child| yield key, child }
+        else
+          children.each
+        end
+      end
+
+      def has_children?
+        !children.empty?
+      end
+
+      def has_child?(key)
+        children.include?(key)
+      end
+
+      def child_at(key)
+        children[key]
+      end
+
+      def add_child(key, node)
+        @children[key] = node
+      end
+
+      def has_value?
+        value != NO_VALUE
+      end
+    end
+
+  end
+end

data/lib/abroad/serializers/xml/android_serializer.rb

@@ -0,0 +1,143 @@
+require 'htmlentities'
+require 'ext/htmlentities/android_xml_encoder'
+
+module Abroad
+  module Serializers
+    module Xml
+
+      class AndroidSerializer < XmlSerializer
+        PLURAL_FORMS = %w(
+          zero one two few many other
+        )
+
+        attr_reader :plurals, :arrays
+
+        def after_initialize
+          writer.open_tag('resources')
+          @plurals = Hash.new { |hash, key| hash[key] = {} }
+          @arrays = Hash.new { |hash, key| hash[key] = {} }
+        end
+
+        def write_raw(text)
+          stream.write(text)
+        end
+
+        def write_key_value(key, value)
+          key_parts = split_key(key)
+
+          case detect_string_type(key_parts)
+            when :plural
+              record_plural(key_parts, value)
+            when :array_element
+              record_array_element(key_parts, value)
+            else
+              write_string(key, value)
+          end
+        end
+
+        def flush
+          write_recorded_plurals
+          write_recorded_arrays
+          writer.flush
+          stream.flush
+        end
+
+        private
+
+        def split_key(key)
+          if idx = key.rindex('.')
+            [key[0..(idx - 1)], key[(idx + 1)..-1]]
+          else
+            # this case should never happen
+            [key, '']
+          end
+        end
+
+        def record_plural(key_parts, value)
+          plurals[key_parts.first][key_parts.last] = value
+        end
+
+        def record_array_element(key_parts, value)
+          arrays[key_parts.first][key_parts.last.to_i] = value
+        end
+
+        def write_plural(name, plural_forms)
+          writer.open_tag(:plurals, name: name)
+
+          plural_forms.each_pair do |quantity, value|
+            writer.open_single_line_tag(:item, quantity: quantity)
+            write_text(value)
+            writer.close_tag
+          end
+
+          writer.close_tag
+        end
+
+        def write_array(name, array_elements)
+          writer.open_tag(:'string-array', name: name)
+          count = array_elements.keys.max
+
+          (0..count).each do |i|
+            writer.open_single_line_tag(:item)
+            write_text(array_elements[i] || '')
+            writer.close_tag
+          end
+
+          writer.close_tag
+        end
+
+        def write_recorded_plurals
+          plurals.each_pair do |name, plural_forms|
+            write_plural(name, plural_forms)
+          end
+        end
+
+        def write_recorded_arrays
+          arrays.each_pair do |name, array_elements|
+            write_array(name, array_elements)
+          end
+        end
+
+        def write_string(key, value)
+          writer.open_single_line_tag(:string, name: key)
+          write_text(value)
+          writer.close_tag
+        end
+
+        def write_text(text)
+          escaped_text = escape(text)
+          writer.write_text(
+            escaped_text, escape: false
+          )
+        end
+
+        def detect_string_type(key_parts)
+          last_part = key_parts.last
+
+          if PLURAL_FORMS.include?(last_part)
+            :plural
+          elsif last_part =~ /\A[\d]+\z/
+            :array_element
+          else
+            :string
+          end
+        end
+
+        def escape(text)
+          text.gsub!("\n", "\\n")                    # escape literal newlines
+          text.gsub!("\r", "\\r")                    # escape literal carriage returns
+          text.gsub!("\t", "\\t")                    # escape literal tabs
+          text.gsub!(/([^\\]?)(')/) { "#{$1}\\'" }   # escape single quotes
+          text.gsub!(/([^\\]?)(")/) { "#{$1}\\\"" }  # escape double quotes
+
+          coder.encode(text)
+        end
+
+        def coder
+          @coder ||= HTMLEntities::AndroidXmlEncoder.new
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/xml/xml_serializer.rb

@@ -0,0 +1,23 @@
+require 'xml-write-stream'
+
+module Abroad
+  module Serializers
+    module Xml
+
+      class XmlSerializer < Serializer
+        attr_reader :writer
+
+        def initialize(stream, locale, encoding = Encoding::UTF_8)
+          super
+          @writer = XmlWriteStream.from_stream(stream)
+          writer.write_header(encoding: encoding.to_s)
+          after_initialize
+        end
+
+        def after_initialize
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/xml.rb

@@ -0,0 +1,8 @@
+module Abroad
+  module Serializers
+    module Xml
+      autoload :AndroidSerializer, 'abroad/serializers/xml/android_serializer'
+      autoload :XmlSerializer,     'abroad/serializers/xml/xml_serializer'
+    end
+  end
+end

data/lib/abroad/serializers/yaml/rails_serializer.rb

@@ -0,0 +1,110 @@
+module Abroad
+  module Serializers
+    module Yaml
+
+      class RailsSerializer < YamlSerializer
+        attr_reader :trie
+
+        def initialize(stream, locale, encoding = Encoding::UTF_8)
+          super
+          @trie = Abroad::Serializers::Trie.new
+        end
+
+        def write_raw(text)
+          writer.stream.write(text)
+        end
+
+        def write_key_value(key, value)
+          key_parts = split_key(key)
+          encoded_value = value.encode(encoding)
+          trie.add(key_parts, encoded_value)
+        end
+
+        def flush
+          writer.write_map(locale)
+          write_node(trie.root, locale)
+          writer.flush
+          stream.flush
+        end
+
+        private
+
+        def split_key(key)
+          # Doesn't allow dots to come before spaces or at the end of the key.
+          # Uses regex negative lookahead, that's what the (?!) sections are.
+          # Examples:
+          # 'timezones.Solomon Is.'     => ['timezones', 'Solomon Is.']
+          # 'timezones.Solomon Is.foo'  => ['timezones', 'Solomon Is', 'foo']
+          # 'timezones.Solomon Is..foo' => ['timezones', 'Solomon Is.', 'foo']
+          key.split(/\.(?!\s)(?!\z)(?!\.)/)
+        end
+
+        # depth-first
+        def write_node(node, parent_key)
+          if node
+            if node.has_children?
+              if children_are_sequence(node)
+                write_sequence(node, parent_key)
+              else
+                write_map(node, parent_key)
+              end
+            elsif node.has_value?
+              write_value(node, parent_key)
+            end
+          else
+            write_value(node, parent_key)
+          end
+        end
+
+        def write_value(node, parent_key)
+          value = node ? node.value : ''
+          if writer.in_map?
+            writer.write_key_value(parent_key, value)
+          else
+            writer.write_element(value)
+          end
+        end
+
+        def write_map(node, parent_key)
+          if writer.in_map?
+            writer.write_map(parent_key)
+          else
+            writer.write_map
+          end
+
+          node.each_child do |key, child|
+            write_node(child, key)
+          end
+
+          writer.close_map
+        end
+
+        def write_sequence(node, parent_key)
+          if writer.in_map?
+            writer.write_sequence(parent_key)
+          else
+            writer.write_sequence
+          end
+
+          generate_sequence(node).each do |element|
+            write_node(element, nil)
+          end
+
+          writer.close_sequence
+        end
+
+        def children_are_sequence(node)
+          node.children.all? { |key, _| key =~ /\A[\d]+\z/ }
+        end
+
+        def generate_sequence(node)
+          keys = node.children.keys.map(&:to_i)
+          keys.each_with_object(Array.new(keys.max)) do |idx, arr|
+            arr[idx] = node.children[idx.to_s]
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/yaml/yaml_serializer.rb

@@ -0,0 +1,19 @@
+require 'yaml'
+require 'yaml-write-stream'
+
+module Abroad
+  module Serializers
+    module Yaml
+
+      class YamlSerializer < Serializer
+        attr_reader :writer
+
+        def initialize(stream, locale, encoding = Encoding::UTF_8)
+          super
+          @writer = YamlWriteStream.from_stream(stream, encoding)
+        end
+      end
+
+    end
+  end
+end

data/lib/abroad/serializers/yaml.rb

@@ -0,0 +1,8 @@
+module Abroad
+  module Serializers
+    module Yaml
+      autoload :RailsSerializer, 'abroad/serializers/yaml/rails_serializer'
+      autoload :YamlSerializer,  'abroad/serializers/yaml/yaml_serializer'
+    end
+  end
+end

data/lib/abroad/serializers.rb

@@ -0,0 +1,29 @@
+module Abroad
+  module Serializers
+    autoload :Json,       'abroad/serializers/json'
+    autoload :Serializer, 'abroad/serializers/serializer'
+    autoload :Trie,       'abroad/serializers/trie'
+    autoload :Xml,        'abroad/serializers/xml'
+    autoload :Yaml,       'abroad/serializers/yaml'
+
+    class << self
+      def register(id, klass)
+        registered[id] = klass
+      end
+
+      def get(id)
+        registered[id]
+      end
+
+      def available
+        registered.keys
+      end
+
+      private
+
+      def registered
+        @registered ||= {}
+      end
+    end
+  end
+end

data/lib/abroad/version.rb

@@ -0,0 +1,3 @@
+module Abroad
+  VERSION = '1.0.0'
+end

data/lib/abroad.rb

@@ -0,0 +1,37 @@
+module Abroad
+  class SyntaxError < StandardError; end
+
+  autoload :Extractors,  'abroad/extractors'
+  autoload :Serializers, 'abroad/serializers'
+
+  Extractors.register('yaml/rails', Extractors::Yaml::RailsExtractor)
+  Extractors.register('yaml/dotted-key', Extractors::Yaml::DottedKeyExtractor)
+  Extractors.register('json/key-value', Extractors::Json::KeyValueExtractor)
+  Extractors.register('xml/android', Extractors::Xml::AndroidExtractor)
+
+  Serializers.register('yaml/rails', Serializers::Yaml::RailsSerializer)
+  Serializers.register('json/key-value', Serializers::Json::KeyValueSerializer)
+  Serializers.register('xml/android', Serializers::Xml::AndroidSerializer)
+
+  class << self
+    def extractors
+      Extractors.available
+    end
+
+    def extractor(id)
+      Extractors.get(id)
+    end
+
+    def serializers
+      Serializers.available
+    end
+
+    def serializer(id)
+      Serializers.get(id)
+    end
+
+    def jruby?
+      RUBY_ENGINE == 'jruby'
+    end
+  end
+end

data/lib/ext/htmlentities/android_xml_decoder.rb

@@ -0,0 +1,15 @@
+require 'htmlentities'
+
+class HTMLEntities
+  MAPPINGS['android_xml'] = MAPPINGS['xhtml1'].dup.tap do |mappings|
+    mappings.delete('apos')
+  end
+
+  FLAVORS << 'android_xml'
+
+  class AndroidXmlDecoder < Decoder
+    def initialize
+      super('android_xml')
+    end
+  end
+end

data/lib/ext/htmlentities/android_xml_encoder.rb

@@ -0,0 +1,23 @@
+require 'htmlentities'
+
+class HTMLEntities
+  MAPPINGS['android_xml'] = MAPPINGS['xhtml1'].dup.tap do |mappings|
+    mappings.delete('apos')
+  end
+
+  FLAVORS << 'android_xml'
+
+  class AndroidXmlEncoder < Encoder
+    def initialize(instructions = [])
+      super('android_xml', instructions)
+    end
+
+    private
+
+    # had to adjust this so that single and double quotes don't get turned into
+    # entities (they're escaped by hand with backslashes)
+    def basic_entity_regexp
+      @basic_entity_regexp ||= /[&]/
+    end
+  end
+end