i18nema 0.0.1

data/ext/i18nema/yamlbyte.h

@@ -0,0 +1,171 @@
+/*  yamlbyte.h
+ *
+ *  The YAML bytecode "C" interface header file.   See the YAML bytecode
+ *  reference for bytecode sequence rules and for the meaning of each
+ *  bytecode.
+ */
+
+#ifndef YAMLBYTE_H
+#define YAMLBYTE_H
+#include <stddef.h>
+
+/* define what a character is */
+typedef unsigned char yamlbyte_utf8_t;
+typedef unsigned short yamlbyte_utf16_t;
+#ifdef YAMLBYTE_UTF8
+  #ifdef YAMLBYTE_UTF16
+    #error Must only define YAMLBYTE_UTF8 or YAMLBYTE_UTF16
+  #endif
+  typedef yamlbyte_utf8_t yamlbyte_char_t;
+#else
+  #ifdef YAMLBYTE_UTF16
+    typedef yamlbyte_utf16_t yamlbyte_char_t;
+  #else
+    #error Must define YAMLBYTE_UTF8 or YAMLBYTE_UTF16
+  #endif
+#endif
+
+/* specify list of bytecodes */
+#define YAMLBYTE_FINISH          ((yamlbyte_char_t) 0)
+#define YAMLBYTE_DOCUMENT        ((yamlbyte_char_t)'D')
+#define YAMLBYTE_DIRECTIVE       ((yamlbyte_char_t)'V')
+#define YAMLBYTE_PAUSE           ((yamlbyte_char_t)'P')
+#define YAMLBYTE_MAPPING         ((yamlbyte_char_t)'M')
+#define YAMLBYTE_SEQUENCE        ((yamlbyte_char_t)'Q')
+#define YAMLBYTE_END_BRANCH      ((yamlbyte_char_t)'E')
+#define YAMLBYTE_SCALAR          ((yamlbyte_char_t)'S')
+#define YAMLBYTE_CONTINUE        ((yamlbyte_char_t)'C')
+#define YAMLBYTE_NEWLINE         ((yamlbyte_char_t)'N')
+#define YAMLBYTE_NULLCHAR        ((yamlbyte_char_t)'Z')
+#define YAMLBYTE_ANCHOR          ((yamlbyte_char_t)'A')
+#define YAMLBYTE_ALIAS           ((yamlbyte_char_t)'R')
+#define YAMLBYTE_TRANSFER        ((yamlbyte_char_t)'T')
+/* formatting bytecodes */
+#define YAMLBYTE_COMMENT         ((yamlbyte_char_t)'c')
+#define YAMLBYTE_INDENT          ((yamlbyte_char_t)'i')
+#define YAMLBYTE_STYLE           ((yamlbyte_char_t)'s')
+/* other bytecodes */
+#define YAMLBYTE_LINE_NUMBER     ((yamlbyte_char_t)'#')
+#define YAMLBYTE_WHOLE_SCALAR    ((yamlbyte_char_t)'<')
+#define YAMLBYTE_NOTICE          ((yamlbyte_char_t)'!')
+#define YAMLBYTE_SPAN            ((yamlbyte_char_t)')')
+#define YAMLBYTE_ALLOC           ((yamlbyte_char_t)'@')
+
+/* second level style bytecodes, ie "s>" */
+#define YAMLBYTE_FLOW            ((yamlbyte_char_t)'>')
+#define YAMLBYTE_LITERAL         ((yamlbyte_char_t)'|')
+#define YAMLBYTE_BLOCK           ((yamlbyte_char_t)'b')
+#define YAMLBYTE_PLAIN           ((yamlbyte_char_t)'p')
+#define YAMLBYTE_INLINE_MAPPING  ((yamlbyte_char_t)'{')
+#define YAMLBYTE_INLINE_SEQUENCE ((yamlbyte_char_t)'[')
+#define YAMLBYTE_SINGLE_QUOTED   ((yamlbyte_char_t)39)
+#define YAMLBYTE_DOUBLE_QUOTED   ((yamlbyte_char_t)'"')
+
+/*
+ * The "C" API has two variants, one based on instructions,
+ * with events delivered via pointers; and the other one
+ * is character based where one or more instructions are
+ * serialized into a buffer.
+ *
+ * Note: In the instruction based API, WHOLE_SCALAR does
+ *       not have the '<here' marshalling stuff.
+ */
+
+typedef void * yamlbyte_consumer_t;
+typedef void * yamlbyte_producer_t;
+
+/* push and pull APIs need a way to communicate results */
+typedef enum {
+    YAMLBYTE_OK          = 0,     /* proceed                        */
+    YAMLBYTE_E_MEMORY    = 'M',   /* could not allocate memory      */
+    YAMLBYTE_E_READ      = 'R',   /* input stream read error        */
+    YAMLBYTE_E_WRITE     = 'W',   /* output stream write error      */
+    YAMLBYTE_E_OTHER     = '?',   /* some other error condition     */
+    YAMLBYTE_E_PARSE     = 'P',   /* parse error, check bytecodes   */
+    YAMLBYTE_MAX
+} yamlbyte_result_t;
+
+typedef const yamlbyte_char_t *yamlbyte_buff_t;
+
+/*
+ *  The "Instruction" API
+ */
+
+typedef struct yaml_instruction {
+    yamlbyte_char_t bytecode;
+    yamlbyte_buff_t start;
+    yamlbyte_buff_t finish;  /* open range, *finish is _not_ part */
+} *yamlbyte_inst_t;
+
+/* producer pushes the instruction with one bytecode event to the
+ * consumer; if the consumer's result is not YAMLBYTE_OK, then
+ * the producer should stop */
+typedef
+  yamlbyte_result_t
+   (*yamlbyte_push_t)(
+     yamlbyte_consumer_t self,
+     yamlbyte_inst_t  inst
+   );
+
+/* consumer pulls a bytecode instruction from the producer; in this
+ * case the instruction (and is buffer) are owned by the producer and
+ * will remain valid till the pull function is called once again;
+ * if the instruction is NULL, then there are no more results; and
+ * it is important to call the pull function till it returns NULL so
+ * that the producer can clean up its memory allocations */
+typedef
+   yamlbyte_result_t
+    (*yamlbyte_pull_t)(
+      yamlbyte_producer_t self,
+      yamlbyte_inst_t *inst   /* to be filled in by the producer */
+    );
+
+/*
+ *  Buffer based API
+ */
+
+/* producer pushes a null terminated buffer filled with one or more
+ * bytecode events to the consumer; if the consumer's result is not
+ * YAMLBYTE_OK, then the producer should stop */
+typedef
+  yamlbyte_result_t
+   (*yamlbyte_pushbuff_t)(
+     yamlbyte_consumer_t self,
+     yamlbyte_buff_t  buff
+   );
+
+/* consumer pulls bytecode events from the producer; in this case
+ * the buffer is owned by the producer, and will remain valid till
+ * the pull function is called once again; if the buffer pointer
+ * is set to NULL, then there are no more results; it is important
+ * to call the pull function till it returns NULL so that the
+ * producer can clean up its memory allocations */
+typedef
+   yamlbyte_result_t
+    (*yamlbyte_pullbuff_t)(
+      yamlbyte_producer_t self,
+      yamlbyte_buff_t *buff   /* to be filled in by the producer */
+    );
+
+/* convert a pull interface to a push interface; the reverse process
+ * requires threads and thus is language dependent */
+#define YAMLBYTE_PULL2PUSH(pull,producer,push,consumer,result)       \
+    do {                                                         \
+        yamlbyte_pullbuff_t _pull = (pull);                              \
+        yamlbyte_pushbuff_t _push = (push);                              \
+        yamlbyte_result_t _result = YAMLBYTE_OK;                         \
+        yamlbyte_producer_t _producer = (producer);                  \
+        yamlbyte_consumer_t _consumer = (consumer);                  \
+        while(1) {                                               \
+            yamlbyte_buff_t buff = NULL;                           \
+            _result = _pull(_producer,&buff);                    \
+            if(YAMLBYTE_OK != result || NULL == buff)                \
+                break;                                           \
+            _result = _push(_consumer,buff);                     \
+            if(YAMLBYTE_OK != result)                                \
+                break;                                           \
+        }                                                        \
+        (result) = _result;                                      \
+    } while(0)
+
+#endif

data/lib/i18nema/core_ext/hash.rb

@@ -0,0 +1,8 @@
+class Hash
+  def deep_stringify_keys
+    Hash[map{ |key, value|
+      value = value.deep_stringify_keys if value.is_a?(Hash)
+      [key.to_s, value]
+    }]
+  end unless Hash.method_defined?(:deep_stringify_keys)
+end

data/lib/i18nema.rb

@@ -0,0 +1,42 @@
+require 'syck'
+require 'i18n'
+require File.dirname(__FILE__) + '/i18nema/core_ext/hash'
+require File.dirname(__FILE__) + '/i18nema/i18nema'
+
+module I18nema
+  class Backend
+    include I18n::Backend::Base
+
+    def store_translations(locale, data, options = {})
+      # TODO: make this moar awesome
+      @initialized = true
+      load_yml_string({locale => data}.deep_stringify_keys.to_yaml)
+    end
+
+    def init_translations
+      load_translations
+      @initialized = true
+    end
+
+  protected
+    def load_file(filename)
+      type = File.extname(filename).tr('.', '').downcase
+      raise I18n::UnknownFileType.new(type, filename) unless type == "yml"
+      load_yml(filename)
+    end
+
+    def load_yml(filename)
+      load_yml_string File.read(filename)
+    end
+
+    def initialized?
+      @initialized
+    end
+
+    def lookup(locale, key, scope = [], options = {})
+      init_translations unless initialized?
+      keys = I18n.normalize_keys(locale, key, scope, options[:separator])
+      direct_lookup(*keys)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,5 @@
+require 'bundler/setup'
+Bundler.require(:default)
+
+require 'test/unit'
+require 'i18nema'

data/test/i18nema_test.rb

@@ -0,0 +1,78 @@
+require 'helper'
+require 'yaml'
+
+class I18nemaTest < Test::Unit::TestCase
+  def setup
+    @data = {
+      foo: {
+        bar: "lol"
+      },
+      baz: %w{
+        asdf
+        qwerty
+      }
+    }.deep_stringify_keys
+    @backend = I18nema::Backend.new
+    @backend.store_translations :en, @data
+  end
+
+  def test_yaml_parity
+    assert_equal({"en" => @data}, @backend.direct_lookup)
+  end
+
+  def test_scoping
+    assert_equal({"bar" => "lol"},
+                 @backend.direct_lookup("en", "foo"))
+    assert_equal "lol",
+                 @backend.direct_lookup("en", "foo", "bar")
+    assert_equal nil,
+                 @backend.direct_lookup("poo")
+  end
+
+  def test_merging
+    @backend.store_translations :en, foo: "replaced!", wat: "added!"
+    assert_equal "replaced!",
+                 @backend.direct_lookup("en", "foo")
+    assert_equal ["asdf", "qwerty"],
+                 @backend.direct_lookup("en", "baz")
+    assert_equal "added!",
+                 @backend.direct_lookup("en", "wat")
+  end
+
+  def test_reload
+    @backend.reload!
+    assert_equal({}, @backend.direct_lookup)
+  end
+
+  def test_available_locales
+    @backend.store_translations :es, foo: "hola"
+    assert_equal ['en', 'es'],
+                 @backend.available_locales.map(&:to_s).sort
+  end
+
+  def test_invalid_yml
+    backend = I18nema::Backend.new
+
+    exception = assert_raise(I18nema::Backend::LoadError) {
+      backend.load_yml_string("string")
+    }
+    assert_equal("root yml node is not a hash", exception.message)
+    assert_equal({}, backend.direct_lookup)
+
+    # FIXME ... ruby syck does this differently, and we get a non
+    # i_object_t as the root node, causing delete_object to asplode when
+    # it tries to free a garbage pointer
+    #
+    #exception = assert_raise(I18nema::Backend::LoadError) {
+    #  backend.load_yml_string("en:\n  foo: \"lol\"\n\tbar: notabs!")
+    #}
+    #assert_match(/TAB found in your indentation/, exception.message)
+    #assert_equal({}, backend.direct_lookup)
+
+    exception = assert_raise(I18nema::Backend::LoadError) {
+      backend.load_yml_string("en:\n  &a [*a]")
+    }
+    assert_match(/bad anchor `a'/, exception.message)
+    assert_equal({}, backend.direct_lookup)
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: i18nema
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jon Jensen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: drop-in replacement for I18n::Backend::Simple for faster lookups and
+  quicker gc runs. translations are stored outside of the ruby heap
+email: jon@instructure.com.com
+executables: []
+extensions:
+- ext/i18nema/extconf.rb
+- ext/i18nema/mkrf_conf.rb
+extra_rdoc_files: []
+files:
+- Rakefile
+- README.md
+- ext/i18nema/i18nema.c
+- ext/i18nema/extconf.h
+- ext/i18nema/gram.h
+- ext/i18nema/syck.h
+- ext/i18nema/uthash.h
+- ext/i18nema/yamlbyte.h
+- ext/i18nema/extconf.rb
+- ext/i18nema/mkrf_conf.rb
+- lib/i18nema/core_ext/hash.rb
+- lib/i18nema.rb
+- test/helper.rb
+- test/i18nema_test.rb
+homepage: http://github.com/instructure/i18nema
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.5
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: fast i18n backend that doesn't stop up the garbage collector
+test_files: []
+has_rdoc: