tml 5.2.7 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a77252a4c27d765aeb0ba967924eacbf0034f7be
-  data.tar.gz: ff61535795e423a7f5b5b256dffb0c70f6e6b30f
+  metadata.gz: 4a84984e64ec22f712cec34434ceba677ccb757a
+  data.tar.gz: 91eaaef9c8621cc7459bb76b7e2abfaece3b702e
 SHA512:
-  metadata.gz: 1ad94fca1129718cf1bfe61ae5b8aafb8d343020d2233fbfd0320d5131f7eee94cc8c31483c1ab5d722d671db29fcd21f2645650f9057727e15bc9852a714f92
-  data.tar.gz: dc5d46be009e0b9701782340e1169b31dd279a6285965913de55ee90d157dbbab3df381fe0df35a48589a4808a9769305d99e799b2df5503559d0f17101c3604
+  metadata.gz: b82d16a5ed538b5c53e458d5c46a999eb80622812de47fb0e325a6b8e3123f564195c81a86e9741f694f7c1d9067a1b64e9241b3429f93db8485834a36d9ae24
+  data.tar.gz: 196079f8c475c04383978418e96dacf95d91d3ed7cf89266a147eefc7b6de0f714881ac7cc690f34359321729879f64db4bc141630695c45e626e520dc085979

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015 Translation Exchange, Inc
+Copyright (c) 2016 Translation Exchange, Inc
 
 MIT License
 

data/README.md

@@ -209,7 +209,7 @@ Links
 
 * Register on TranslationExchange.com: http://translationexchange.com
 
-* Read TranslationExchange's documentation: http://translationexchange.com/docs
+* Read TranslationExchange's documentation: http://docs.translationexchange.com
 
 * Follow TranslationExchange on Twitter: https://twitter.com/translationx
 
@@ -221,7 +221,7 @@ Links
 Copyright and license
 ==================
 
-Copyright (c) 2015 Translation Exchange, Inc
+Copyright (c) 2016 Translation Exchange, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/tml/api/client.rb

@@ -39,31 +39,37 @@ class Tml::Api::Client < Tml::Base
 
   attributes :application
 
-  # get results from api
+  # get results from API
   def results(path, params = {}, opts = {})
     get(path, params, opts)['results']
   end
 
+  # get from API
   def get(path, params = {}, opts = {})
     api(path, params, opts.merge(:method => :get))
   end
 
+  # post to API
   def post(path, params = {}, opts = {})
     api(path, params, opts.merge(:method => :post))
   end
 
+  # put to API
   def put(path, params = {}, opts = {})
     api(path, params, opts.merge(:method => :put))
   end
 
+  # delete from API
   def delete(path, params = {}, opts = {})
     api(path, params, opts.merge(:method => :delete))
   end
 
+  # checks if there are any API errors
   def self.error?(data)
     not data['error'].nil?
   end
 
+  # API connection
   def connection
     @connection ||= Faraday.new(:url => application.host) do |faraday|
       faraday.request(:url_encoded)               # form-encode POST params
@@ -72,22 +78,34 @@ class Tml::Api::Client < Tml::Base
     end
   end
 
+  # get cache version from CDN
   def get_cache_version
-    execute_request('projects/current/version', {}, {:raw => true})
+    data = get_from_cdn('version', {t: Time.now.to_i}, {public: true, uncompressed: true})
+
+    unless data
+      Tml.logger.debug('No releases have been published yet')
+      return '0'
+    end
+
+    data['version']
   end
 
+  # verify cache version
   def verify_cache_version
-    return if Tml.cache.version and Tml.cache.version != 'undefined'
+    return if Tml.cache.version.defined?
+
+    current_version = Tml.cache.version.fetch
 
-    current_version = Tml.cache.fetch_version
     if current_version == 'undefined'
-      Tml.cache.store_version(get_cache_version)
+      Tml.cache.version.store(get_cache_version)
     else
-      Tml.cache.version = current_version
+      Tml.cache.version.set(current_version)
     end
-    Tml.logger.info("Version: #{Tml.cache.version}")
+
+    Tml.logger.info("Cache Version: #{Tml.cache.version}")
   end
 
+  # cdn_connection
   def cdn_connection
     @cdn_connection ||= Faraday.new(:url => application.cdn_host) do |faraday|
       faraday.request(:url_encoded)               # form-encode POST params
@@ -95,11 +113,21 @@ class Tml::Api::Client < Tml::Base
     end
   end
 
+  # get from the CDN
   def get_from_cdn(key, params = {}, opts = {})
-    return nil if Tml.cache.version == 'undefined' || Tml.cache.version.to_s == '0'
+    if Tml.cache.version.invalid? and key != 'version'
+      return nil
+    end
 
     response = nil
-    cdn_path = "/#{application.key}/#{Tml.cache.version}/#{key}.json.gz"
+    cdn_path = "/#{application.key}"
+
+    if key == 'version'
+      cdn_path += "/#{key}.json"
+    else
+      cdn_path += "/#{Tml.cache.version.to_s}/#{key}.json#{opts[:uncompressed] ? '' : '.gz'}"
+    end
+
     trace_api_call(cdn_path, params, opts.merge(:host => application.cdn_host)) do
       begin
         response = cdn_connection.get do |request|
@@ -116,8 +144,12 @@ class Tml::Api::Client < Tml::Base
     compressed_data = response.body
     return if compressed_data.nil? or compressed_data == ''
 
-    data = Zlib::GzipReader.new(StringIO.new(compressed_data.to_s)).read
-    Tml.logger.debug("Compressed: #{compressed_data.length} Uncompressed: #{data.length}")
+    data = compressed_data
+
+    unless opts[:uncompressed]
+      data = Zlib::GzipReader.new(StringIO.new(compressed_data.to_s)).read
+      Tml.logger.debug("Compressed: #{compressed_data.length} Uncompressed: #{data.length}")
+    end
 
     begin
       data = JSON.parse(data)
@@ -128,39 +160,52 @@ class Tml::Api::Client < Tml::Base
     data
   end
 
-  def live_translations_request?(cache_key)
-    return if cache_key.blank?
-    return unless Tml.session.block_option(:live)
-    cache_key.index('sources') or cache_key.index('keys')
+  # access token
+  def access_token
+    application.token
+  end
+
+  # should the API go to live server
+  def live_api_request?
+    # if no access token, never use live mode
+    return false if access_token.blank?
+
+    # if block is specifically asking for it or inline mode is activated
+    Tml.session.inline_mode? or Tml.session.block_option(:live)
   end
 
-  def enable_cache?(opts)
+  # checks if cache is enable
+  def cache_enabled?(opts)
+    # only gets ever get cached
     return false unless opts[:method] == :get
     return false if opts[:cache_key].nil?
     return false unless Tml.cache.enabled?
-    return false if live_translations_request?(opts[:cache_key])
-    return false if Tml.session.inline_mode?
     true
   end
 
+  # checks mode and cache, and fetches data
   def api(path, params = {}, opts = {})
-    # inline mode should always bypass API calls
-    # get request uses local cache, then CDN, the API
-    if enable_cache?(opts)
-      verify_cache_version
-      data = Tml.cache.fetch(opts[:cache_key]) do
-        if Tml.cache.read_only?
-          nil
-        else
-          get_from_cdn(opts[:cache_key]) || execute_request(path, params, opts)
-        end
-      end
-      process_response(data, opts)
-    else
-      process_response(execute_request(path, params, opts), opts)
+    # inline mode should always use API calls
+    if live_api_request?
+      return process_response(execute_request(path, params, opts), opts)
+    end
+
+    return unless cache_enabled?(opts)
+
+    # ensure the cache version is not outdated
+    verify_cache_version
+
+    return if Tml.cache.version.invalid?
+
+    # get request uses local cache, then CDN
+    data = Tml.cache.fetch(opts[:cache_key]) do
+      Tml.cache.read_only? ? nil : get_from_cdn(opts[:cache_key])
     end
+
+    process_response(data, opts)
   end
 
+  # paginates through API results
   def paginate(path, params = {}, opts = {})
     data = get(path, params, opts.merge({'raw' => true}))
 
@@ -181,31 +226,35 @@ class Tml::Api::Client < Tml::Base
     end
   end
 
+  # prepares API path
   def prepare_api_path(path)
-    return path if path.index('oauth')
     return path if path.match(/^https?:\/\//)
     "#{API_PATH}#{path[0] == '/' ? '' : '/'}#{path}"
   end
 
+  # prepares request
   def prepare_request(request, path, params)
-    request.options.timeout = 30
-    request.options.open_timeout = 20
+    request.options.timeout             = Tml.config.api_client[:timeout]
+    request.options.open_timeout        = Tml.config.api_client[:open_timeout]
     request.headers['User-Agent']       = "tml-ruby v#{Tml::VERSION} (Faraday v#{Faraday::VERSION})"
     request.headers['Accept']           = 'application/json'
     request.headers['Accept-Encoding']  = 'gzip, deflate'
     request.url(path, params)
   end
 
+  # execute API request
   def execute_request(path, params = {}, opts = {})
     response = nil
     error = nil
 
-    # oauth path is separate from versioned APIs
     path = prepare_api_path(path)
-    params = params.merge(:access_token => application.token) unless path.index('oauth')
+
+    unless opts[:public]
+      params = params.merge(:access_token => access_token)
+    end
 
     if opts[:method] == :post
-      params = params.merge(:api_key => application.key)
+      params = params.merge(:app_id => application.key)
     end
 
     @compressed = false
@@ -235,7 +284,7 @@ class Tml::Api::Client < Tml::Base
       raise Tml::Exception.new("Error: #{response.body}")
     end
 
-    if @compressed
+    if @compressed and (not opts[:uncompressed])
       compressed_data = response.body
       return if compressed_data.nil? or compressed_data == ''
 
@@ -260,11 +309,13 @@ class Tml::Api::Client < Tml::Base
     data
   end
 
+  # get object class from options
   def object_class(opts)
     return unless opts[:class]
     opts[:class].is_a?(String) ? opts[:class].constantize : opts[:class]
   end
 
+  # process API response
   def process_response(data, opts)
     return nil if data.nil?
     return data if opts[:raw] or opts[:raw_json]
@@ -283,18 +334,22 @@ class Tml::Api::Client < Tml::Base
     object_class(opts).new(data.merge(opts[:attributes] || {}))
   end
 
+  # convert params to query
   def to_query(hash)
     query = []
     hash.each do |key, value|
-      query << "#{key}=#{value}"
+      query << "#{key.to_s}=#{value.to_s}"
     end
     query.join('&')
   end
 
+  # trace api call for logging
   def trace_api_call(path, params, opts = {})
-    #[:client_secret, :access_token].each do |param|
-    #  params = params.merge(param => "##filtered##") if params[param]
-    #end
+    if Tml.config.logger[:secure]
+      [:access_token].each do |param|
+        params = params.merge(param => "##filtered##") if params[param]
+      end
+    end
 
     path = "#{path[0] == '/' ? '' : '/'}#{path}"
 

data/lib/tml/application.rb

@@ -35,9 +35,10 @@ require 'faraday'
 class Tml::Application < Tml::Base
 
   CDN_HOST = 'https://cdn.translationexchange.com'
+  # CDN_HOST = 'https://trex-snapshots.s3-us-west-1.amazonaws.com'
   API_HOST = 'https://api.translationexchange.com'
 
-  attributes :host, :id, :key, :access_token,  :name, :description, :threshold, :default_locale, :default_level, :tools
+  attributes :host, :id, :key, :access_token,  :name, :description, :threshold, :default_locale, :default_level
   has_many :features, :languages, :languages_by_locale, :sources, :tokens, :css, :shortcuts, :translations, :extensions
   has_many :ignored_keys
 
@@ -65,7 +66,7 @@ class Tml::Application < Tml::Base
 
   # Fetches application definition from the service
   def fetch
-    data = api_client.get('projects/current/definition',{
+    data = api_client.get("projects/#{key}/definition",{
       locale: Tml.session.current_locale,
       source: Tml.session.current_source,
       ignored: true
@@ -76,7 +77,8 @@ class Tml::Application < Tml::Base
     if data
       update_attributes(data)
     else
-      Tml.logger.debug('Cache enabled but no data is provided.')
+      add_language(Tml.config.default_language)
+      Tml.logger.debug('Cache is disabled or no data has been cached.')
     end
 
     self
@@ -177,16 +179,6 @@ class Tml::Application < Tml::Base
     @locales ||= languages.collect{|lang| lang.locale}
   end
 
-  # Returns tools data
-  def tools
-    @attributes[:tools] || {}
-  end
-
-  # Returns asset url
-  def url_for(path)
-    "#{tools['assets']}#{path}"
-  end
-
   # Returns source by key
   def source(key, locale)
     self.sources ||= {}
@@ -196,7 +188,7 @@ class Tml::Application < Tml::Base
     ).fetch_translations(locale)
   end
 
-  # Verifies current source path
+  # Verify current source path
   def verify_source_path(source_key, source_path)
     return if Tml.cache.enabled? and not Tml.session.inline_mode?
     return if extensions.nil? or extensions['sources'].nil?
@@ -205,14 +197,17 @@ class Tml::Application < Tml::Base
     @missing_keys_by_sources[source_path] ||= {}
   end
 
+  # Register missing keys
   def register_missing_key(source_key, tkey)
     return if Tml.cache.enabled? and not Tml.session.inline_mode?
+    source_key = source_key.to_s
     @missing_keys_by_sources ||= {}
     @missing_keys_by_sources[source_key] ||= {}
     @missing_keys_by_sources[source_key][tkey.key] ||= tkey
     submit_missing_keys if Tml.config.submit_missing_keys_realtime
   end
 
+  # Register keys
   def register_keys(keys)
     params = []
     keys.each do |source_key, keys|
@@ -228,12 +223,14 @@ class Tml::Application < Tml::Base
     Tml.logger.error(e.backtrace)
   end
 
+  # Submit missing translation keys
   def submit_missing_keys
     return if @missing_keys_by_sources.nil? or @missing_keys_by_sources.empty?
     register_keys(@missing_keys_by_sources)
     @missing_keys_by_sources = nil
   end
 
+  # Reset translation cache
   def reset_translation_cache
     self.sources = {}
     self.translations = {}
@@ -241,18 +238,20 @@ class Tml::Application < Tml::Base
     @missing_keys_by_sources  = nil
   end
 
+  # Check if a key is ignored
   def ignored_key?(key)
     return false if ignored_keys.nil?
     not ignored_keys.index(key).nil?
   end
 
+  # Fetch translations from API
   def fetch_translations(locale)
     self.translations ||= {}
     self.translations[locale] ||= begin
       results = Tml.cache.fetch(Tml::Application.translations_cache_key(locale)) do
         data = {}
         unless Tml.cache.read_only?
-          data = api_client.get('projects/current/translations', :all => true, :ignored => true, :raw_json => true)
+          data = api_client.get("projects/#{key}/translations", :all => true, :ignored => true, :raw_json => true)
         end
         data
       end
@@ -280,6 +279,7 @@ class Tml::Application < Tml::Base
     {}
   end
 
+  # Cache translations within application object
   def cache_translations(locale, key, new_translations)
     self.translations ||= {}
     self.translations[locale] ||= {}
@@ -292,11 +292,13 @@ class Tml::Application < Tml::Base
     end
   end
 
+  # Get application cached translations
   def cached_translations(locale, key)
     return unless self.translations and self.translations[locale]
     self.translations[locale][key]
   end
 
+  # Debug translations
   def debug_translations
     return 'no translations' unless self.translations
     self.translations.each do |locale, keys|
@@ -308,24 +310,24 @@ class Tml::Application < Tml::Base
     end
   end
 
+  # Get default decoration token
   def default_decoration_token(token)
     hash_value(tokens, "decoration.#{token.to_s}")
   end
 
+  # Get default data token
   def default_data_token(token)
     hash_value(tokens, "data.#{token.to_s}")
   end
 
+  # Check if feature is enabled
   def feature_enabled?(key)
     hash_value(features, key.to_s)
   end
 
+  # Create API client
   def api_client
-    @api_client ||= Tml.config.api_client_class.new(application: self)
-  end
-
-  def postoffice
-    @postoffice ||= Tml::Api::PostOffice.new(application: self)
+    @api_client ||= Tml.config.api_client[:class].new(application: self)
   end
 
 end

data/lib/tml/cache.rb

@@ -30,9 +30,13 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-module Tml
+require 'rubygems'
+require 'rubygems/package'
+require 'open-uri'
+require 'zlib'
+require 'fileutils'
 
-  CACHE_VERSION_KEY = 'current_version'
+module Tml
 
   def self.cache
     @cache ||= begin
@@ -48,41 +52,19 @@ module Tml
 
   class Cache
 
-    # Returns current cache version
+    # version object
     def version
-      @version
-    end
-
-    # sets the current version
-    def version=(new_version)
-      @version = new_version
+      @version ||= Tml::CacheVersion.new(self)
     end
 
     # resets current version
     def reset_version
-      @version = nil
+      version.reset
     end
 
     # upgrade current version
     def upgrade_version
-      store(CACHE_VERSION_KEY, {'version' => 'undefined'})
-      reset_version
-    end
-
-    # fetches the version from the cache
-    def fetch_version
-      @version ||= begin
-        v = fetch(CACHE_VERSION_KEY) do
-          {'version' => Tml.config.cache[:version] || 'undefined'}
-        end
-        v.is_a?(Hash) ? v['version'] : v
-      end
-    end
-
-    # stores the current version back in cache
-    def store_version(new_version)
-      @version = new_version
-      store(CACHE_VERSION_KEY, {'version' => new_version})
+      version.upgrade
     end
 
     # checks if Tml is enabled
@@ -114,12 +96,12 @@ module Tml
     # namespace of each cache key
     def namespace
       return '#' if Tml.config.disabled?
-      Tml.config.cache[:namespace] || Tml.config.access_token[0..5]
+      Tml.config.cache[:namespace] || Tml.config.application[:key][0..5]
     end
 
     # versioned name of cache key
     def versioned_key(key, opts = {})
-      "tml_#{namespace}#{CACHE_VERSION_KEY == key ? '' : "_v#{version}"}_#{key}"
+      version.versioned_key(key, namespace)
     end
 
     # fetches key from cache
@@ -148,6 +130,105 @@ module Tml
       # do nothing
     end
 
+    def extract_version(app, version = nil)
+      if version
+        Tml.cache.version.set(version.to_s)
+      else
+        version_data = app.api_client.get_from_cdn('version', {t: Time.now.to_i}, {uncompressed: true})
+
+        unless version_data
+          Tml.logger.debug('No releases have been generated yet. Please visit your Dashboard and publish translations.')
+          return
+        end
+
+        Tml.cache.version.set(version_data['version'])
+      end
+    end
+
+    # warmup cache
+    def warmup(version = nil)
+      t0 = Time.now
+      Tml.logger = Logger.new(STDOUT)
+
+      Tml.logger.debug('Starting cache warmup...')
+      app = Tml::Application.new(key: Tml.config.application[:key])
+      extract_version(app, version)
+
+      Tml.logger.debug("Warming Up Version: #{Tml.cache.version}")
+
+      application = app.api_client.get_from_cdn('application', {t: Time.now.to_i})
+      Tml.cache.store(Tml::Application.cache_key, application)
+
+      sources = app.api_client.get_from_cdn('sources', {t: Time.now.to_i}, {uncompressed: true})
+
+      application['languages'].each do |lang|
+        locale = lang['locale']
+        language = app.api_client.get_from_cdn("#{locale}/language", {t: Time.now.to_i})
+        Tml.cache.store(Tml::Language.cache_key(locale), language)
+
+        sources.each do |src|
+          source = app.api_client.get_from_cdn("#{locale}/sources/#{src}", {t: Time.now.to_i})
+          Tml.cache.store(Tml::Source.cache_key(locale, src), source)
+        end
+      end
+
+      t1 = Time.now
+      Tml.logger.debug("Cache warmup took #{t1-t0}s")
+    end
+
+    # default cache path
+    def default_cache_path
+      @cache_path ||= begin
+        path = Tml.config.cache[:path]
+        path ||= 'config/tml'
+        FileUtils.mkdir_p(path)
+        FileUtils.chmod(0777, path)
+        path
+      end
+    end
+
+    # downloads cache from the CDN
+    def download(cache_path = default_cache_path, version = nil)
+      t0 = Time.now
+      Tml.logger = Logger.new(STDOUT)
+
+      Tml.logger.debug('Starting cache download...')
+      app = Tml::Application.new(key: Tml.config.application[:key])
+      extract_version(app, version)
+
+      Tml.logger.debug("Downloading Version: #{Tml.cache.version}")
+
+      archive_name = "#{Tml.cache.version}.tar.gz"
+      path = "#{cache_path}/#{archive_name}"
+      url = "#{app.cdn_host}/#{Tml.config.application[:key]}/#{archive_name}"
+
+      Tml.logger.debug("Downloading cache file: #{url}")
+      open(path, 'wb') do |file|
+        file << open(url).read
+      end
+
+      Tml.logger.debug('Extracting cache file...')
+      version_path = "#{cache_path}/#{Tml.cache.version}"
+      Tml::Utils.untar(Tml::Utils.ungzip(File.new(path)), version_path)
+      Tml.logger.debug("Cache has been stored in #{version_path}")
+
+      File.unlink(path)
+
+      begin
+        current_path = 'current'
+        FileUtils.chdir(cache_path)
+        FileUtils.rm(current_path) if File.exist?(current_path)
+        FileUtils.ln_s(Tml.cache.version.to_s, current_path)
+        Tml.logger.debug("The new version #{Tml.cache.version} has been marked as current")
+      rescue Exception => ex
+        Tml.logger.debug("Could not generate current symlink to the cache path: #{ex.message}")
+      end
+
+      t1 = Time.now
+      Tml.logger.debug("Cache download took #{t1-t0}s")
+    end
+
+    # remove extensions
     def strip_extensions(data)
       if data.is_a?(Hash)
         data = data.dup