ghost_reader 1.0.0

data/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea
+*~
+*.swp
+\#*
+.#*

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm ruby-1.8.7@ghost_reader

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ghost_reader.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/README.md

@@ -0,0 +1,35 @@
+GhostReader
+===========
+
+i18n backend to ghost_writer service
+
+## Usage
+
+add a folowing to `config/initializers/ghost_reader.rb`
+
+    I18n.backend=GhostReader::Backend.new("HTTP_URL_TO_GHOST_SERVER",
+                  :default_backend=>I18n.backend, :wait_time=>30,
+                  :trace => Proc.new do |message|
+                    Rails.logger.debug message
+                  end)
+
+### wait_time
+The 'wait_time' is the minimum time in seconds after which reached a change
+from the Ghost_Writer Ghost_Client. A low value minimizes the delay,
+a high value minimizes the network-traffic.
+Default-Value is 30
+
+### default_backend
+The Ghost_reader tries to find default-values for not found translations and
+posts them to the server together with the statistical data.
+
+### max_packet_size
+For preventing Errors on receiving Server splits the ghost-reader the posts
+in parts with max max_packet_size count of keys
+
+### trace
+Proc for logging connection handling to Server
+
+
+
+

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler'
+require 'rake'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new :spec
+task :default => :spec

data/doc/ARCH.md

@@ -0,0 +1,54 @@
+Client
+======
+
+    class Client
+    
+      # returns a Head with three keys
+      #   :timestamp (the value of last-modified header)
+      #   :data (a nested Hash of translations)
+      #   :status (the reponse status)
+      def initial_request
+      end
+    
+      # returns true if redirected, false otherwise
+      def reporting_request(data)
+      end
+    
+      # returns a Head with three keys
+      #   :timestamp (the value of last-modified header)
+      #   :data (a nested Hash of translations)
+      #   :status (the reponse status)
+      def incremental_request
+      end
+    
+    end
+
+Initializer
+===========
+
+    options = {
+      :host => '',
+      :update_interval => 1.minute,
+      :reset_interval => 15.minutes,
+      :fallback => I18n.backend,
+      :api_key => '91885ca9ec4feb9b2ed2423cdbdeda32'
+    }
+    I18n.backend = GhostReader::I18nBackend.new(options).start_agents
+
+
+Custom Initializer for Development
+==================================
+
+    require File.expand_path(File.join(%w(.. .. .. .. ghost_reader lib ghost_reader)), __FILE__)
+    
+    config = {
+      :report_interval => 5, # secs
+      :retrieval_interval => 10, # secs
+      :fallback => I18n.backend,
+      :logfile => File.join(Rails.root, %w(log ghostwriter.log)),
+      :service => {
+        :api_key => '9d07cf6d805ea2951383c9ed76db762e' # Ghost Dummy Project
+      }
+    }
+    
+    I18n.backend = GhostReader::Backend.new(config).spawn_agents

data/doc/SPEC.md

@@ -0,0 +1,138 @@
+GhostReader/Writer protocol (initial draft)
+===========================================
+
+# Purpose
+
+The ghost/writer communication protocol should define the process for
+efficient exchange of the missing i18n translations from the reader and
+completed translations from the writer.
+
+# Specification
+
+## Prerequisites
+
+* The client/server communication should use HTTPS
+* The communication should use JSON for exchanging data
+* Identification should be performed by use of an API key
+* Authentication/Authorization are not yet addressed
+
+## Use cases
+
+* There are no requests specific to a locale. All request will update
+  or return translations for multiple locales.
+* I18n keys may occur in two different forms:
+  - aggregated (string), e.g. `"this.is.a.sample.key"`
+  - nested (hash), e.g. (in JSON) `{"this":{"is":{"a":{"sample":{"key":null}}}}}`
+
+### Application start (initial request)
+
+* On application start a request should be made from server to send the
+  already completed translations
+  - Use rails caching for case when multiple instances of the application are started
+  - This request has to be nonlocking/async, and must not hang or
+    crash the server if it fails
+  - The server performs caching on this request. (Note: Because of
+    different output formats it might make sense to caching on object
+    level.)
+  - The server must set the "Last-Modified" HTTP header.
+  - Request scheme: (1) `GET https://ghostwriter/api/<APIKEY>/translations`
+  - The client will track the "Last-Modified" time for use in
+    'incremental requests'.
+  - This request will also respond to YAML & CSV for exporting.
+
+### Client reports missing translations (reporting request)
+
+* The client should gather a collection of all missing translations.
+  - If a translation is missing the lookup will cascade into other
+    sources (I18n backends, like YAML files). (Note: This cannot be
+    achieved by chaining backends since fallbacks will not be
+    propagated.)
+  - If the lookup yields a result the result will reported along with
+    the key as a default value.
+
+* The client should POST data for the missing translations.
+  - the server must respond with a Redirect-After-Post redirecting to (1)
+  - Request scheme: (2) `POST https://ghostwriter/api/<APIKEY>/translations`
+
+* The server will validate the keys and create or update untranslated
+  entries.
+
+### Client recieves updated translations (incremental request)
+
+* The client should GET data for updated translations
+  - The client must set the "If-Modified-Since" HTTP header. (Otherwise
+    the request equals the inital GET and all of the translations are sent.)
+  - The server will only send the transaltions that where updated
+    between the potint in time denoted by the "If-Modified-Since"
+    header and the time of the request (now).
+  - The server will set the "Last-Modified" HTTP header.
+  - The server will NOT perform any caching.
+  - Request scheme: (3) `GET https://ghostwriter/api/<APIKEY>/translations`,
+    with "If-Modified-Since" HTTP header set
+* The client will merge the recieved data into it's current pool of
+  translations.
+  - Thereby the client will overwrite any conflicting translations
+    with the newly recieved data.
+* The client will track the "Last-Modified" header for future requests.
+
+### Error handling
+
+* The client should log the error and retry the request.
+* After a number of retries the client should inform the administrator
+  (through email, sms... etc.) that there is a problem.
+
+## Data model definition
+
+### Request
+
+* The request data should contain the following:
+  - locale code
+  - the i18n keys (aggregated) which where requested but have no
+    translation
+  - the default values if fallback lookups yielded a result
+  - a count, indicating how often they were requested (can be used
+    as a proxy variable for importance)
+  - timestamp when the last request was made
+    (as HTTP header "Last-Modified" -> "If-Modified-Since")
+* Sample Request, reporting missing (JSON)
+
+```
+{"sample.key_1":{"en":{"count":42,"default":"Sample translation 1."}},
+"sample.key_2":{"en":{"count":23,"default":"Sample translation 2."}}}
+```
+
+### Response
+
+* The response data should contain the following
+  - keys (nested) and their translations, nested in locales
+  - timestamp, when response was made ("Last-Modifed" HTTP header)
+* Sample Response (JSON)
+
+```
+{"en":{"sample":{"key_1":"Sample translation 1.","key_2":"Sample translation 2."}}}
+
+```
+
+## Lookup
+
+The client as a I18n backend will lookup translations in multiple sources.
+
+1. Pool (maybe there is a better name for it)
+  - local
+  - maybe use Rails cache, to share pool over multiple instances
+2. GhostWriter API
+  - Asynchronously!
+  - As long as translations are missing, the client will poll the
+    server for updated translations and merge any newly recieved
+    translations into the local Pool.
+3. Fallback
+  - SimpleBackend (locale files)
+  - or whatever is used by the developers during coding
+
+# Addendum
+
+## HTTP header date formats are specified here
+
+* http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3
+* http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.29
+* e.g. in ruby strftime format '%a, %d %b %Y %H:%M:%S %Z'

data/doc/request_mockups.rb

@@ -0,0 +1,40 @@
+require 'rubygems'
+require 'excon'
+require 'json'
+
+# Excon.ssl_verify_peer = false
+
+address = 'http://0.0.0.0:3000/api/91885ca9ec4feb9b2ed2423cdbdeda32/translations.json'
+excon = Excon.new(address)
+puts
+
+puts "(1) Initial request... (GET without If-Modified-Since)"
+response = excon.get
+puts
+puts "  Status:         #{response.status}"
+puts "  Body size:      #{response.body.size}"
+@last_modified = response.get_header('Last-Modified')
+puts "  Last-Modified:  #{@last_modified}"
+puts
+
+puts "(2) Reporting request... (POST)"
+data = {
+  "sample.key_1" => {"en" => {"count" => 42, "default" => "Sample translation 1."}},
+  "sample.key_2" => {"en" => {"count" => 23, "default" => "Sample translation 2."}}
+}
+response = excon.post(:body => "data=#{data.to_json}")
+puts
+puts "  Status:         #{response.status}"
+puts
+
+puts "Sleeping a second to avoid automatic 304"
+sleep 1
+puts
+
+puts "(3) Incremental request... (GET with If-Modified-Since)"
+headers = { 'If-Modified-Since' => @last_modified }
+response = excon.get(:headers => headers)
+puts
+puts "  Status:         #{response.status}"
+puts "  Body size:      #{response.body.size}"
+puts

data/ghost_reader.gemspec

@@ -0,0 +1,35 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "ghost_reader/version"
+
+Gem::Specification.new do |s|
+  s.name        = "ghost_reader"
+  s.version     = GhostReader::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Andreas König", "Phil Hofmann"]
+  s.email       = ["koa@panter.ch", "phil@branch14.org"]
+  s.homepage    = "https://github.com/branch14/ghost_reader"
+  s.summary     = %q{i18n backend to ghost_writer service}
+  s.description = %q{Loads I18n-Yaml-Files via http and exchanges statistical data
+and updates with the ghost_server}
+
+  s.rubyforge_project = "ghost_reader"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency('i18n')
+  s.add_dependency('json')
+  s.add_dependency('excon')
+
+  s.add_development_dependency('ruby-debug')
+  s.add_development_dependency('guard')
+  s.add_development_dependency('guard-rspec')
+
+  s.add_development_dependency('rack')
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec')
+  s.add_development_dependency('mongrel')
+  s.add_development_dependency('actionpack', '3.0.7')
+end

data/lib/ghost_reader/backend.rb

@@ -0,0 +1,161 @@
+require 'logger'
+require 'ostruct'
+require 'i18n/backend/transliterator' # i18n/backend/base fails to require this
+require 'i18n/backend/base'
+require 'i18n/backend/memoize'
+require 'i18n/backend/flatten'
+
+module GhostReader
+  class Backend
+
+    module DebugLookup
+      def lookup(*args)
+        config.logger.debug "Lookup: #{args.inspect}"
+        # debugger
+        super
+      end
+    end
+
+    module Implementation
+
+      attr_accessor :config, :missings
+
+      # for options see code of default_config
+      def initialize(conf={})
+        self.config = OpenStruct.new(default_config.merge(conf))
+        yield(config) if block_given?
+        config.logger = Logger.new(config.logfile || STDOUT)
+        config.logger.level = config.log_level || Logger::WARN
+        config.service[:logger] ||= config.logger
+        config.client = Client.new(config.service)
+        config.logger.info "Initialized GhostReader backend."
+      end
+
+      def spawn_agents
+        config.logger.debug "GhostReader spawning agents."
+        spawn_retriever
+        spawn_reporter
+        config.logger.debug "GhostReader spawned its agents."
+        self
+      end
+
+      protected
+
+      # this won't be called if memoize kicks in
+      def lookup(locale, key, scope = [], options = {})
+        raise 'no fallback given' if config.fallback.nil?
+        config.fallback.translate(locale, key, options).tap do |result|
+          # TODO results which are hashes need to be tracked disaggregated
+          track({ key => { locale => { 'default' => result } } }) unless result.is_a?(Hash)
+        end
+      end
+
+      def track(missings)
+        return if self.missings.nil? # not yet initialized
+        self.missings.deep_merge!(missings)
+      end
+
+      def memoize_merge!(data, options={ :method => :merge! })
+        flattend = flatten_translations_for_all_locales(data)
+        symbolized_flattend = symbolize_keys(flattend)
+        memoized_lookup.send(options[:method], symbolized_flattend)
+      end
+
+      # performs initial and incremental requests
+      def spawn_retriever
+        config.logger.debug "Spawning retriever."
+        Thread.new do
+          begin
+            config.logger.debug "Performing initial request."
+            response = config.client.initial_request
+            memoize_merge! response[:data]
+            self.missings = {} # initialized
+            config.logger.info "Initial request successfull."
+            until false
+              begin
+                sleep config.retrieval_interval
+                response = config.client.incremental_request
+                if response[:status] == 200
+                  config.logger.info "Incremental request with data."
+                  config.logger.debug "Data: #{response[:data].inspect}"
+                  memoize_merge! response[:data], :method => :deep_merge!
+                else
+                  config.logger.debug "Incremental request, but no data."
+                end
+              rescue
+                config.logger.error "Exception in retriever loop: #{ex}"
+              end
+            end
+          rescue => ex
+            config.logger.error "Exception in retriever thread: #{ex}"
+            config.logger.debug ex.backtrace.join("\n")
+          end
+        end
+      end
+
+      # performs reporting requests
+      def spawn_reporter
+        config.logger.debug "Spawning reporter."
+        Thread.new do
+          until false
+            begin
+              sleep config.report_interval
+              unless self.missings.nil?
+                unless self.missings.empty?
+                  config.logger.info "Reporting request with #{self.missings.keys.size} missings."
+                  config.client.reporting_request(missings)
+                  missings.clear
+                else
+                  config.logger.debug "Reporting request omitted, nothing to report."
+                end
+              else
+                config.logger.debug "Reporting request omitted, not yet initialized, waiting for intial request."
+              end
+            rescue => ex
+              config.logger.error "Exception in reporter thread: #{ex}" 
+              config.logger.debug ex.backtrace.join("\n")
+            end
+          end
+        end
+      end
+
+      # a wrapper for I18n::Backend::Flatten#flatten_translations
+      def flatten_translations_for_all_locales(data)
+        data.inject({}) do |result, key_value|
+          begin
+            key, value = key_value
+            result.merge key => flatten_translations(key, value, true, false)
+          rescue ArgumentError => ae
+            config.logger.error "Error: #{ae}"
+            result
+          end
+        end
+      end
+
+      def symbolize_keys(hash)
+        hash.each.inject({}) do |symbolized_hash, key_value|
+          key, value = key_value
+          symbolized_hash.merge!({key.to_sym, value})
+        end
+      end
+
+      def default_config
+        {
+          :retrieval_interval => 15,
+          :report_interval => 10,
+          :fallback => nil, # a I18n::Backend (mandatory)
+          :logfile => nil, # a path
+          :log_level => nil, # Log level, the options are Config::(FATAL, ERROR, WARN, INFO and DEBUG)           (http://www.ruby-doc.org/stdlib/libdoc/logger/rdoc/Logger.html)
+          :service => {} # nested hash, see GhostReader::Client#default_config
+        }
+      end
+    end
+
+    include I18n::Backend::Base
+    include Implementation
+    include I18n::Backend::Memoize # provides @memoized_lookup
+    include I18n::Backend::Flatten # provides #flatten_translations
+    include DebugLookup
+  end
+end
+

data/lib/ghost_reader/client.rb

@@ -0,0 +1,91 @@
+require 'excon'
+require 'json'
+
+module GhostReader
+  class Client
+
+    attr_accessor :config, :last_modified
+
+    def initialize(conf=nil)
+      self.config = OpenStruct.new(default_config.merge(conf || {}))
+      config.logger ||= Logger.new(config.logfile || STDOUT)
+    end
+
+    # returns a Head with three keys
+    #   :timestamp (the value of last-modified header)
+    #   :data (a nested Hash of translations)
+    #   :status (the reponse status)
+    def initial_request
+      response = connect_with_retry
+      self.last_modified = response.get_header('Last-Modified')
+      build_head(response)
+    end
+
+    # returns true if redirected, false otherwise
+    def reporting_request(data)
+      response = connect_with_retry(:post, :body => "data=#{data.to_json}")
+      config.logger.error "Reporting request not redirected" unless response.status == 302
+      { :status => response.status }
+    end
+
+    # returns a Head with three keys
+    #   :timestamp (the value of last-modified header)
+    #   :data (a nested Hash of translations)
+    #   :status (the reponse status)
+    def incremental_request
+      headers = { 'If-Modified-Since' => self.last_modified }
+      response = connect_with_retry(:get, :headers => headers)
+      self.last_modified = response.get_header('Last-Modified') if response.status == 200
+      build_head(response)
+    end
+
+    # this is just a wrapper to have a log message when the field is set
+    def last_modified=(value)
+      config.logger.debug "Last-Modified: #{value}"
+      @last_modified = value
+    end
+
+    private
+
+    def build_head(excon_response)
+      { :status => excon_response.status }.tap do |result|
+        result[:data] = JSON.parse(excon_response.body) if excon_response.status == 200
+      end
+    end
+
+    def service
+      @service ||= Excon.new(address)
+    end
+
+    def address
+      raise 'no api_key provided' if config.api_key.nil?
+      @address ||= config.uri.sub(':api_key', config.api_key)
+    end
+
+    # Wrapper method for retrying the connection
+    #   :method - http method (post and get supported at the moment)
+    #   :params - parameters sent to the service (excon)
+    def connect_with_retry(method = :get, params = {})
+      retries = self.config.connection_retries
+      while (retries > 0) do
+        response = service.send(method, params)
+
+        if response.status == 408
+          config.logger.error "Connection time-out. Retrying... #{retries}"
+          retries -= 1
+        else
+          retries = 0 # There is no timeout, no need to retry
+        end
+      end
+      response
+    end
+
+    def default_config
+      {
+        :uri => 'http://ghost.panter.ch/api/:api_key/translations.json',
+        :api_key => nil,
+        :connection_retries => 3
+      }
+    end
+  end
+end

data/lib/ghost_reader/version.rb

@@ -0,0 +1,3 @@
+module GhostReader
+  VERSION = "1.0.0"
+end

data/lib/ghost_reader.rb

@@ -0,0 +1,4 @@
+%w(backend client).each do |f|
+  require File.expand_path(File.join(%w(.. ghost_reader), f), __FILE__)
+end
+

data/lib/tasks/ghost_reader.rake

@@ -0,0 +1,62 @@
+namespace :ghost_reader do
+
+  desc "Fetch newest translations from ghost-writer and overwrite the local translations"
+  task :fetch => :environment do
+    unless I18n.backend.respond_to? :load_yaml_from_ghostwriter
+      raise "ERROR: Ghostwriter is not configured as I18n.backend"
+    end
+
+    begin
+      puts "Loading data from Ghostwriter and delete old translations"
+      yaml_data = I18n.backend.load_yaml_from_ghostwriter
+    rescue Exception => e
+      abort e.message
+    end
+
+    yaml_data.each_pair do |key,value|
+      outfile = Rails.root.join("config", "locales",
+                                "#{key.to_s}.yml")
+      begin
+        puts "Deleting old translations: #{outfile}"
+        File.delete(outfile) if File.exists?(outfile)
+      rescue Exception => e
+        abort "Couldn't delete file: #{e.message}"
+      end
+
+      begin
+        puts "Writing new translations: #{outfile}"
+        File.open(outfile, "w") do |yaml_file|
+          yaml_file.write({key => value}.to_yaml)
+        end
+      rescue Exception => e
+        abort "Couldn't write file: #{e.message}"
+      end
+    end
+  end
+
+  desc "Push all locally configured translations to ghost-writer"
+  task :push => :environment do
+    unless I18n.backend.respond_to? :push_all_backend_data
+      raise "ERROR: Ghostwriter is not configured as I18n.backend"
+    end
+    puts "Pushing data to Ghostwriter"
+    begin
+      I18n.backend.push_all_backend_data
+    rescue Exception => e
+      abort e.message
+    end
+  end
+
+  desc "Install default initializer for ghost reader"
+  task :install => :environment do
+
+    # TODO: this should work only when the ghost_reader is introduced as gem
+    infile = 'templates/ghost_reader.rb'
+    outdir = Rails.root.join("config", "initializers")
+
+    puts "Installing ghost_reader.rb initializer..."
+    FileUtils.copy_file(infile, outdir)
+    puts "Done."
+  end
+
+end

data/lib/tasks/templates/ghost_reader.rb

@@ -0,0 +1,14 @@
+# NOTE: We won't need this probably because the ghost_reader will be loaded as a gem
+# require File.expand_path(File.join(%w(.. .. .. .. ghost_reader lib ghost_reader)), __FILE__)
+
+config = {
+  :report_interval => 5, # secs
+  :retrieval_interval => 10, # secs
+  :fallback => I18n.backend,
+  :logfile => File.join(Rails.root, %w(log ghostwriter.log)),
+  :service => {
+    :api_key => '9d07cf6d805ea2951383c9ed76db762e' # Ghost Dummy Project
+  }
+}
+
+I18n.backend = GhostReader::Backend.new(config).spawn_agents