lit 0.4.0.pre.alpha.4 → 1.1.0

data/lib/lit/cloud_translation/providers/base.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Lit::CloudTranslation::Providers
+  # Abstract base class for cloud translation providers, providing a skeleton
+  # for the provider's functionality (mainly the #translate method) as well as
+  # a configuration management mechanism.
+  class Base
+    attr_reader :config
+
+    def initialize(config)
+      default_config.each do |key, value|
+        config[key] ||= value
+      end
+      @config = config
+    end
+
+    # Translates a given text from a given language to a different one.
+    # @param [String, Array] text The text (or array of texts) to translate
+    # @param [Symbol, String] from The language to translate from. If not given,
+    #   auto-detection will be attempted.
+    # @param [Symbol, String] to The language to translate to.
+    # @param [Hash] opts Additional, provider-specific optional parameters.
+    def translate(text:, from: nil, to:, **opts) # rubocop:disable Lint/UnusedMethodArgument, Metrics/LineLength
+      raise NotImplementedError
+    end
+
+    private
+
+    # Loads specific information from environment variables or other sources
+    # as the default configuartion for the translation provider.
+    #
+    # This can be overridden using `Lit::CloudTranslation.configure`.
+    def default_config
+      {}
+    end
+
+    private_class_method :new
+
+    class << self
+      # Using the provider object's singleton instance, translates a given text from
+      # a given language to a different one.
+      # @param [String, Array] text The text (or array of texts) to translate
+      # @param [Symbol, String] from The language to translate from. If not given,
+      #   auto-detection will be attempted.
+      # @param [Symbol, String] to The language to translate to.
+      # @param [Hash] opts Additional, provider-specific optional parameters.
+      def translate(text:, from: nil, to:, **opts)
+        instance.translate(text: text, from: from, to: to, **opts)
+      end
+
+      def configure
+        yield config if block_given?
+      end
+
+      private
+
+      def instance
+        @instance ||= new(config)
+      end
+
+      def config
+        @config ||= OpenStruct.new
+      end
+    end
+  end
+end

data/lib/lit/cloud_translation/providers/google.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+begin
+  require 'google/cloud/translate'
+rescue LoadError
+  raise StandardError, 'You need to add "gem \'google-cloud-translate\'" to your Gemfile to support Google Cloud Translation'
+end
+
+module Lit::CloudTranslation::Providers
+  # Google Cloud Translation API provider for Lit translation suggestions.
+  #
+  # Configuration:
+  #
+  #   require 'lit/cloud_translation/providers/google'
+  #
+  #   Lit::CloudTranslation.provider = Lit::CloudTranslation::Providers::Google
+  #
+  #   # Service account configuration can be given via a file pointed to by
+  #   # ENV['GOOGLE_TRANSLATE_API_KEYFILE'] (see here:
+  #   # https://cloud.google.com/iam/docs/creating-managing-service-account-keys)
+  #   #
+  #   # Instead of providing the keyfile, credentials can be given using
+  #   # GOOGLE_TRANSLATE_API_<element> environment variables, where e.g.
+  #   # the GOOGLE_TRANSLATE_API_PROJECT_ID variable corresponds to the
+  #   # `project_id` element of your credentials. Typically, only the following
+  #   # variables are mandatory:
+  #   # * GOOGLE_TRANSLATE_API_PROJECT_ID
+  #   # * GOOGLE_TRANSLATE_API_PRIVATE_KEY_ID
+  #   # * GOOGLE_TRANSLATE_API_PRIVATE_KEY (be sure that it contains correct line breaks)
+  #   # * GOOGLE_TRANSLATE_API_CLIENT_EMAIL
+  #   # * GOOGLE_TRANSLATE_API_CLIENT_ID
+  #   #
+  #   # Alternatively, the contents of that file can be given as a Ruby hash
+  #   # and passed like the following (be careful to use secrets or something
+  #   # that prevents exposing private credentials):
+  #
+  #   Lit::CloudTranslation.configure do |config|
+  #     config.keyfile_hash = {
+  #       'type' => 'service_account',
+  #       'project_id' => 'foo',
+  #       'private_key_id' => 'keyid',
+  #       ... # see link above for reference
+  #     }
+  #   end
+  class Google < Base
+    def translate(text:, from: nil, to:, **opts)
+      @client ||=
+        ::Google::Cloud::Translate.new(project_id: config.keyfile_hash['project_id'],
+                                       credentials: config.keyfile_hash)
+      result = @client.translate(sanitize_text(text), from: from, to: to, **opts)
+      unsanitize_text(
+        case result
+        when ::Google::Cloud::Translate::Translation then result.text
+        when Array then result.map(&:text)
+        end
+      )
+    rescue Signet::AuthorizationError => e
+      error_description =
+        'Google credentials error: ' + # rubocop:disable Style/RescueModifier
+        JSON.parse(e.response.body)['error_description'] rescue 'Unknown error'
+      raise ::Lit::CloudTranslation::TranslationError, error_description,
+            cause: e
+    rescue ::Google::Cloud::Error => e
+      raise ::Lit::CloudTranslation::TranslationError, e.message, cause: e
+    end
+
+    private
+
+    def default_config
+      if ENV['GOOGLE_TRANSLATE_API_KEYFILE'].blank?
+        env_keys = ENV.keys.grep(/\AGOOGLE_TRANSLATE_API_/)
+        keyfile_keys = env_keys.map(&:downcase).map { |k| k.gsub('google_translate_api_', '') }
+        keyfile_key_to_env_key_mapping = keyfile_keys.zip(env_keys).to_h
+        return {
+          keyfile_hash: keyfile_key_to_env_key_mapping.transform_values do |env_key|
+            ENV[env_key]
+          end
+        }
+      end
+      { keyfile_hash: JSON.parse(File.read(ENV['GOOGLE_TRANSLATE_API_KEYFILE'])) }
+    rescue JSON::ParserError
+      raise
+    rescue Errno::ENOENT
+      { keyfile_hash: nil }
+    end
+
+    def require_config!
+      return if config.keyfile_hash.present?
+      raise 'GOOGLE_TRANSLATE_API_KEYFILE env or `config.keyfile_hash` not given'
+    end
+
+    def sanitize_text(text_or_array)
+      case text_or_array
+      when String
+        text_or_array.gsub(/%{(.+?)}/, '<code>__LIT__\1__LIT__</code>')
+      when Array
+        text_or_array.map { |s| sanitize_text(s) }
+      when nil
+        ''
+      else
+        raise TypeError
+      end
+    end
+
+    def unsanitize_text(text_or_array)
+      case text_or_array
+      when String
+        text_or_array.gsub(/<code>__LIT__(.+?)__LIT__<\/code>/, '%{\1}')
+      when Array
+        text_or_array.map { |s| unsanitize_text(s) }
+      else
+        raise TypeError
+      end
+    end
+  end
+end

data/lib/lit/cloud_translation/providers/yandex.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'net/http'
+
+module Lit::CloudTranslation::Providers
+  # Yandex Translate API provider for Lit translation suggestions.
+  #
+  # Configuration:
+  #
+  #   require 'lit/cloud_translation/providers/yandex'
+  #
+  #   Lit::CloudTranslation.provider = Lit::CloudTranslation::Providers::Yandeex
+  #
+  #   # API key can be given via ENV['YANDEX_TRANSLATE_API_KEY'].
+  #   #
+  #   # Alternatively, it can be set programmatically after setting provider:
+  #
+  #   Lit::CloudTranslation.configure do |config|
+  #     config.api_key = 'the_api_key'
+  #   end
+  class Yandex < Base
+    def translate(text:, from: nil, to:, **opts) # rubocop:disable Metrics/MethodLength, Metrics/LineLength
+      # puts "api key is: #{config.api_key}"
+      # puts "translating #{text} from #{from} to #{to}"
+      uri = URI('https://translate.yandex.net/api/v1.5/tr.json/translate')
+      params = {
+        key: config.api_key,
+        text: sanitize_text(text),
+        lang: [from, to].compact.join('-'),
+        format: opts[:format],
+        options: opts[:options]
+      }.compact
+      uri.query = URI.encode_www_form(params)
+      res = Net::HTTP.get_response(uri)
+
+      unsanitize_text(
+        case res
+        when Net::HTTPOK
+          translations = JSON.parse(res.body)['text']
+          translations.size == 1 ? translations.first : translations
+        else
+          raise ::Lit::CloudTranslation::TranslationError,
+                (JSON.parse(res.body)['message'] rescue "Unknown error: #{res.body}") # rubocop:disable Style/RescueModifier, Metrics/LineLength
+        end
+      )
+    end
+
+    private
+
+    def default_config
+      { api_key: ENV['YANDEX_TRANSLATE_API_KEY'] }
+    end
+
+    def require_config!
+      return if config.api_key.present?
+      raise 'YANDEX_TRANSLATE_API_KEY env or `config.api_key` not given'
+    end
+
+    def sanitize_text(text_or_array)
+      case text_or_array
+      when String
+        text_or_array.gsub(/%{(.+?)}/, '%{_LIT_\1_LIT_}')
+      when Array
+        text_or_array.map { |s| sanitize_text(s) }
+      when nil
+        ''
+      else
+        raise TypeError
+      end
+    end
+
+    def unsanitize_text(text_or_array)
+      case text_or_array
+      when String
+        text_or_array.gsub(/%{_LIT_(.+?)_LIT_}/, '%{\1}')
+      when Array
+        text_or_array.map { |s| unsanitize_text(s) }
+      else
+        raise TypeError
+      end
+    end
+  end
+end

data/lib/lit/engine.rb

@@ -9,11 +9,11 @@ module Lit
     initializer 'lit.assets.precompile' do |app|
       app.config.assets.precompile += %w[lit/application.css lit/application.js]
       app.config.assets.precompile += %w[lit/lit_frontend.css lit/lit_frontend.js]
-      # add language flags to list of precompiled assets
-      if app.config.i18n.available_locales
-        app.config.i18n.available_locales.each do |l|
-          app.config.assets.precompile << "lit/famfamfam_flags/#{l.to_s[0,2]}.png"
-        end
+    end
+
+    initializer 'lit.reloader' do |app|
+      config.to_prepare do
+        Lit.loader.cache.reset_local_cache if Lit.loader
       end
     end
 
@@ -24,5 +24,11 @@ module Lit
         end
       end
     end
+
+    initializer :append_before_action do
+      ActionController::Base.send :before_action do
+        Thread.current[:lit_thread_cache] = {}
+      end
+    end
   end
 end

data/lib/lit/export.rb

@@ -1,4 +1,5 @@
 require 'csv'
+require 'lit/services/localization_keys_to_hash_service'
 
 module Lit
   class Export
@@ -17,7 +18,7 @@ module Lit
 
       case format
       when :yaml
-        exported_keys = nested_string_keys_to_hash(db_localizations)
+        exported_keys = Lit::LocalizationKeysToHashService.call(db_localizations)
         exported_keys.to_yaml
       when :csv
         relevant_locales = locale_keys.presence || I18n.available_locales.map(&:to_s)
@@ -55,23 +56,6 @@ module Lit
       end
     end
 
-    private_class_method def self.nested_string_keys_to_hash(db_localizations)
-      # http://subtech.g.hatena.ne.jp/cho45/20061122
-      deep_proc = proc do |_k, s, o|
-        if s.is_a?(Hash) && o.is_a?(Hash)
-          next s.merge(o, &deep_proc)
-        end
-        next o
-      end
-      nested_keys = {}
-      db_localizations.sort.each do |k, v|
-        key_parts = k.to_s.split('.')
-        converted = key_parts.reverse.reduce(v) { |a, n| { n => a } }
-        nested_keys.merge!(converted, &deep_proc)
-      end
-      nested_keys
-    end
-
     # This is like Array#transpose but ignores size differences between inner arrays.
     private_class_method def self.transpose(matrix)
       maxlen = matrix.max { |x| x.length }.length

data/lib/lit/i18n_backend.rb

@@ -69,10 +69,20 @@ module Lit
 
       parts = I18n.normalize_keys(locale, key, scope, options[:separator])
       key_with_locale = parts.join('.')
+
+      # we might want to return content later, but we first need to check if it's in cache.
+      # it's important to rememver, that accessing non-existen key modifies cache by creating one
+      had_key = @cache.has_key?(key_with_locale)
+
       # check in cache or in simple backend
       content = @cache[key_with_locale] || super
+
+      # return if content is in cache - it CAN be `nil`
+      return content if had_key && !options[:default]
+
       return content if parts.size <= 1
-      if content.nil? && should_cache?(key_with_locale, options)
+
+      if content.nil? && should_cache?(key_with_locale, options, had_key)
         new_content = @cache.init_key_with_value(key_with_locale, content)
         content = new_content if content.nil? # Content can change when Lit.humanize is true for example
         # so there is no content in cache - it might not be if ie. we're doing
@@ -106,7 +116,7 @@ module Lit
           # it anyway if we return nil, but then it will wrap it also in
           # translation_missing span.
           # Humanizing key should be last resort
-          if content.nil? && Lit.humanize_key && key.match(Lit.humanize_key_ignored).nil?
+          if content.nil? && Lit.humanize_key && Lit.humanize_key_ignored.match(key).nil?
             content = key.to_s.split('.').last.humanize
             if content.present?
               @cache[key_with_locale] = content
@@ -129,7 +139,7 @@ module Lit
         # end
       elsif data.respond_to?(:to_str) || data.is_a?(Array)
         key = ([locale] + scope).join('.')
-        return if startup_process && @cache.keys.member?(key) && Lit.ignore_yaml_on_startup
+        return if startup_process && Lit.ignore_yaml_on_startup && (Thread.current[:lit_cache_keys] || @cache.keys).member?(key)
         @cache.update_locale(key, data, data.is_a?(Array), startup_process)
       elsif data.nil?
         return if startup_process
@@ -139,11 +149,13 @@ module Lit
     end
 
     def load_translations_to_cache
+      Thread.current[:lit_cache_keys] = @cache.keys
       ActiveRecord::Base.transaction do
         (@translations || {}).each do |locale, data|
           store_item(locale, data, [], true) if valid_locale?(locale)
         end
       end
+      Thread.current[:lit_cache_keys] = nil
     end
 
     def init_translations
@@ -155,7 +167,7 @@ module Lit
       # load translations from database to cache
       @cache.load_all_translations
       # load translations from @translations to cache
-      load_translations_to_cache
+      load_translations_to_cache unless Lit.ignore_yaml_on_startup
       @initialized = true
     end
 
@@ -179,9 +191,10 @@ module Lit
       Lit.ignored_keys.any?{ |k| key_without_locale.start_with?(k) }
     end
 
-    def should_cache?(key_with_locale, options)
-      if @cache.has_key?(key_with_locale)
-        return false unless options[:default]
+    # checks if should cache. `had_key` is passed, as once cache has been accesed, it's already modified and key exists
+    def should_cache?(key_with_locale, options, had_key)
+      if had_key
+        return false unless options[:default] && !options[:default].is_a?(Array)
       end
 
       _, key_without_locale = ::Lit::Cache.split_key(key_with_locale)

data/lib/lit/import.rb

@@ -33,7 +33,6 @@ module Lit
           yml = parsed_yaml[locale.to_s]
           Hash[*Lit::Cache.flatten_hash(yml)].each do |key, default_translation|
             next if default_translation.nil? && skip_nil
-            puts key
             upsert(locale, key, default_translation)
           end
         end
@@ -52,7 +51,6 @@ module Lit
         row_translations.each do |locale, value|
           next unless locale_keys.blank? || locale_keys.map(&:to_sym).include?(locale.to_sym)
           next if value.nil? && skip_nil
-          puts key
           upsert(locale, key, value)
         end
       end
@@ -138,17 +136,20 @@ module Lit
         # is the array
         val = value.is_a?(Array) ? [value] : value
         I18n.t(key, default: val)
-        unless @raw
-          # this indicates that this translation already exists
-          existing_translation =
-            Lit::Localization.joins(:locale, :localization_key)
-                             .find_by('localization_key = ? and locale = ?',
-                                      key, locale)
-          if existing_translation
-            existing_translation.update(translated_value: value, is_changed: true)
-            lkey = existing_translation.localization_key
-            lkey.update(is_deleted: false) if lkey.is_deleted
-          end
+
+        # this indicates that this translation already exists
+        existing_translation =
+        Lit::Localization.joins(:locale, :localization_key)
+                         .find_by('localization_key = ? and locale = ?', key, locale)
+
+        return unless existing_translation
+        
+        if @raw
+          existing_translation.update(default_value: value)
+        else
+          existing_translation.update(translated_value: value, is_changed: true)
+          lkey = existing_translation.localization_key
+          lkey.update(is_deleted: false) if lkey.is_deleted
         end
       end
     end
@@ -160,7 +161,7 @@ module Lit
             accu[-1] = [
               row.first,
               *accu[-1].drop(1)
-                       .map { |x| Array.wrap(x) }
+                       .map { |x| Array.wrap(x).presence || [x] }
                        .zip(row.drop(1)).map(&:flatten)
             ]
           end