lit 0.3.3 → 0.4.0.pre.alpha

data/db/migrate/20181017123839_lit_add_is_deleted_to_localization_keys.rb

@@ -0,0 +1,8 @@
+class LitAddIsDeletedToLocalizationKeys < Rails::VERSION::MAJOR >= 5   ?
+                                       ActiveRecord::Migration[4.2] :
+                                       ActiveRecord::Migration
+  def change
+    add_column :lit_localization_keys, :is_deleted, :boolean,
+               default: false, null: false
+  end
+end

data/db/migrate/20181018075955_lit_add_localization_key_is_deleted_to_localization_keys.rb

@@ -0,0 +1,8 @@
+class LitAddLocalizationKeyIsDeletedToLocalizationKeys < Rails::VERSION::MAJOR >= 5   ?
+                                                      ActiveRecord::Migration[4.2] :
+                                                      ActiveRecord::Migration
+  def change
+    add_column :lit_incomming_localizations, :localization_key_is_deleted,
+               :boolean, null: false, default: false
+  end
+end

data/db/migrate/20181030111522_lit_add_is_visited_again_to_localization_keys.rb

@@ -0,0 +1,8 @@
+class LitAddIsVisitedAgainToLocalizationKeys < Rails::VERSION::MAJOR >= 5   ?
+                                               ActiveRecord::Migration[4.2] :
+                                               ActiveRecord::Migration
+  def change
+    add_column :lit_localization_keys, :is_visited_again, :boolean,
+               null: false, default: false
+  end
+end

data/lib/generators/lit/install/templates/initializer.rb

@@ -12,6 +12,10 @@ Lit.authentication_verification = <%= @authentication_verification || 'nil' %>
 # environment
 Lit.key_value_engine = '<%= @key_value_engine %>'
 
+# Redis URL to use when key_value_engine is 'redis'
+# When set to `nil`, it uses the REDIS_URL environment variable.
+# Lit.redis_url = 'redis://redis-server:6379/0'
+
 # Pass extra options to key_value_neinge, ie. prefix for redis (only one
 # supported at the moment)
 # Lit.storage_options = { prefix: "my_project" }
@@ -43,7 +47,7 @@ Lit.set_last_updated_at_upon_creation = true
 
 # Store request info - this will store in cache additional info about request
 # path that triggered translation key to be displayed / accessed
-# For more infor please check README.md
+# For more information please check the README.md
 Lit.store_request_info = false
 
 # Initialize lit

data/lib/lit.rb

@@ -5,6 +5,7 @@ module Lit
   mattr_accessor :authentication_function
   mattr_accessor :authentication_verification
   mattr_accessor :key_value_engine
+  mattr_accessor :redis_url
   mattr_accessor :storage_options
   mattr_accessor :humanize_key
   mattr_accessor :ignored_keys

data/lib/lit/adapters/redis_storage.rb

@@ -8,7 +8,7 @@ module Lit
   end
 
   def determine_redis_provider
-    ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+    Lit.redis_url || ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
   end
 
   class RedisStorage

data/lib/lit/cache.rb

@@ -64,8 +64,8 @@ module Lit
       locale_key, key_without_locale = split_key(key)
       locale = find_locale(locale_key)
       localization = find_localization(locale, key_without_locale, value: value, force_array: force_array, update_value: true)
-      return localization.get_value if startup_process && localization.is_changed?
-      localizations[key] = localization.get_value if localization
+      return localization.translation if startup_process && localization.is_changed?
+      localizations[key] = localization.translation if localization
     end
 
     def update_cache(key, value)
@@ -82,12 +82,12 @@ module Lit
     end
 
     def load_all_translations
-      first = Localization.order(id: :asc).first
-      last = Localization.order(id: :desc).first
+      first = Localization.active.order(id: :asc).first
+      last = Localization.active.order(id: :desc).first
       if !first || (!localizations.has_key?(first.full_key) ||
         !localizations.has_key?(last.full_key))
-        Localization.includes([:locale, :localization_key]).find_each do |l|
-          localizations[l.full_key] = l.get_value
+        Localization.includes(%i[locale localization_key]).active.find_each do |l|
+          localizations[l.full_key] = l.translation
         end
       end
     end
@@ -97,7 +97,7 @@ module Lit
       locale_key, key_without_locale = split_key(key)
       locale = find_locale(locale_key)
       localization = find_localization(locale, key_without_locale, default_fallback: true)
-      localizations[key] = localization.get_value if localization
+      localizations[key] = localization.translation if localization
     end
 
     def delete_key(key)
@@ -126,40 +126,6 @@ module Lit
       @locale_cache[locale_key]
     end
 
-    # this comes directly from copycopter.
-    def export
-      reset
-      localizations_scope = Lit::Localization
-      unless ENV['LOCALES'].blank?
-        locale_keys = ENV['LOCALES'].to_s.split(',') || []
-        locale_ids = Lit::Locale.where(locale: locale_keys).pluck(:id)
-        localizations_scope = localizations_scope.where(locale_id: locale_ids) unless locale_ids.empty?
-      end
-      db_localizations = {}
-      localizations_scope.find_each do |l|
-        db_localizations[l.full_key] = l.get_value
-      end
-      exported_keys = nested_string_keys_to_hash(db_localizations)
-      exported_keys.to_yaml
-    end
-
-    def nested_string_keys_to_hash(db_localizations)
-      # http://subtech.g.hatena.ne.jp/cho45/20061122
-      deep_proc = proc do |_k, s, o|
-        if s.is_a?(Hash) && o.is_a?(Hash)
-          next s.merge(o, &deep_proc)
-        end
-        next o
-      end
-      nested_keys = {}
-      db_localizations.sort.each do |k, v|
-        key_parts = k.to_s.split('.')
-        converted = key_parts.reverse.reduce(v) { |a, n| { n => a } }
-        nested_keys.merge!(converted, &deep_proc)
-      end
-      nested_keys
-    end
-
     def get_global_hits_counter(key)
       @hits_counter['global_hits_counter.' + key]
     end
@@ -190,11 +156,14 @@ module Lit
       return nil if value.is_a?(Hash)
       ActiveRecord::Base.transaction do
         localization_key = find_localization_key(key_without_locale)
-        localization = Lit::Localization.where(locale_id: locale.id). \
-                          where(localization_key_id: localization_key.id).first_or_initialize
+        localization =
+          Lit::Localization.active
+                           .where(locale_id: locale.id)
+                           .where(localization_key_id: localization_key.id)
+                           .first_or_initialize
         if update_value || localization.new_record?
           if value.is_a?(Array)
-            value = parse_array_value(value) unless force_array
+            value = parse_array_value(value, locale) unless force_array
           elsif !value.nil?
             value = parse_value(value, locale)
           else
@@ -204,7 +173,10 @@ module Lit
               value = fallback_to_default(localization_key, localization)
             end
           end
-          localization.update_default_value(value)
+          # Prevent overwriting existing default value with nil.
+          # However, if the localization record is #new_record?, we still need
+          # to insert it with an empty default value.
+          localization.update_default_value(value) if localization.new_record? || value
         end
         return localization
       end
@@ -236,8 +208,8 @@ module Lit
     def find_localization_for_delete(locale, key_without_locale)
       localization_key = find_localization_key_for_delete(key_without_locale)
       return nil unless localization_key
-      Lit::Localization.find_by(locale_id: locale.id,
-                                localization_key_id: localization_key.id)
+      Lit::Localization.active.find_by(locale_id: locale.id,
+                                       localization_key_id: localization_key.id)
     end
 
     def delete_localization(locale, key_without_locale)
@@ -259,9 +231,9 @@ module Lit
         when Symbol then
           lk = Lit::LocalizationKey.where(localization_key: v.to_s).first
           if lk
-            loca = Lit::Localization.where(locale_id: locale.id).
+            loca = Lit::Localization.active.where(locale_id: locale.id).
                         where(localization_key_id: lk.id).first
-            new_value = loca.get_value if loca && loca.get_value.present?
+            new_value = loca.translation if loca && loca.translation.present?
           end
         when String then
           new_value = v
@@ -275,7 +247,7 @@ module Lit
       new_value
     end
 
-    def parse_array_value(value)
+    def parse_array_value(value, locale)
       new_value = nil
       value_clone = value.dup
       while (v = value_clone.shift) && v.present?
@@ -286,10 +258,12 @@ module Lit
     end
 
     def find_localization_key(key_without_locale)
-      unless localization_keys.key?(key_without_locale)
-        find_or_create_localization_key(key_without_locale)
+      if localization_keys.key?(key_without_locale)
+        Lit::LocalizationKey.find_by(
+          id: localization_keys[key_without_locale]
+        ) || find_or_create_localization_key(key_without_locale)
       else
-        Lit::LocalizationKey.find_by(id: localization_keys[key_without_locale]) || find_or_create_localization_key(key_without_locale)
+        find_or_create_localization_key(key_without_locale)
       end
     end
 
@@ -303,7 +277,11 @@ module Lit
     end
 
     def find_or_create_localization_key(key_without_locale)
-      localization_key = Lit::LocalizationKey.where(localization_key: key_without_locale).first_or_create!
+      localization_key = Lit::LocalizationKey.find_or_initialize_by(
+        localization_key: key_without_locale
+      )
+      localization_key.is_visited_again = true if localization_key.is_deleted?
+      localization_key.save! if localization_key.changed?
       localization_keys[key_without_locale] = localization_key.id
       localization_key
     end

data/lib/lit/export.rb

@@ -0,0 +1,80 @@
+require 'csv'
+
+module Lit
+  class Export
+    def self.call(locale_keys:, format:)
+      raise ArgumentError, "format must be yaml or csv" if %i[yaml csv].exclude?(format)
+      Lit.loader.cache.load_all_translations
+      localizations_scope = Lit::Localization.active
+      if locale_keys.present?
+        locale_ids = Lit::Locale.where(locale: locale_keys).pluck(:id)
+        localizations_scope = localizations_scope.where(locale_id: locale_ids) unless locale_ids.empty?
+      end
+      db_localizations = {}
+      localizations_scope.find_each do |l|
+        db_localizations[l.full_key] = l.translation
+      end
+
+      case format
+      when :yaml
+        exported_keys = nested_string_keys_to_hash(db_localizations)
+        exported_keys.to_yaml
+      when :csv
+        relevant_locales = locale_keys.presence || I18n.available_locales.map(&:to_s)
+        CSV.generate do |csv|
+          csv << ['key', *relevant_locales]
+          keys_without_locales = db_localizations.keys.map { |k| k.gsub(/(#{relevant_locales.join('|')})\./, '') }.uniq
+          keys_without_locales.each do |key_without_locale|
+            # Here, we need to determine if we're dealing with an array or a scalar.
+            # In the former case, for simplicity of editing (which is likely the main
+            # intent when exporting translations to CSV), let's make the "array" be simulated
+            # as a number of consecutive rows that have the same key.
+            #
+            # For example:
+            #
+            # key,en
+            # date.abbr_month_names,     <-- in this case it's empty because that array has nothing at [0]
+            # date.abbr_month_names,Jan
+            # date.abbr_month_names,Feb
+            # date.abbr_month_names,Mar
+            # date.abbr_month_names,Apr
+            # date.abbr_month_names,May
+            # ...
+
+            key_localizations_per_locale =
+              relevant_locales.map { |l| Array.wrap(db_localizations["#{l}.#{key_without_locale}"]) }
+            transpose(key_localizations_per_locale).each do |translation_series|
+              csv << [key_without_locale, *translation_series]
+            end
+          end
+        end
+      end
+    end
+
+    private_class_method def self.nested_string_keys_to_hash(db_localizations)
+      # http://subtech.g.hatena.ne.jp/cho45/20061122
+      deep_proc = proc do |_k, s, o|
+        if s.is_a?(Hash) && o.is_a?(Hash)
+          next s.merge(o, &deep_proc)
+        end
+        next o
+      end
+      nested_keys = {}
+      db_localizations.sort.each do |k, v|
+        key_parts = k.to_s.split('.')
+        converted = key_parts.reverse.reduce(v) { |a, n| { n => a } }
+        nested_keys.merge!(converted, &deep_proc)
+      end
+      nested_keys
+    end
+
+    # This is like Array#transpose but ignores size differences between inner arrays.
+    private_class_method def self.transpose(matrix)
+      maxlen = matrix.max { |x| x.length }.length
+      matrix.each do |array|
+        array[maxlen - 1] = nil if array.length < maxlen
+      end
+      matrix.transpose
+    end
+  end
+end

data/lib/lit/i18n_backend.rb

@@ -3,6 +3,7 @@ require 'i18n'
 module Lit
   class I18nBackend
     include I18n::Backend::Simple::Implementation
+    include I18n::Backend::Pluralization
 
     attr_reader :cache
 
@@ -10,10 +11,9 @@ module Lit
       @cache = cache
       @available_locales_cache = nil
       @translations = {}
-      reserved_keys = I18n.const_get :RESERVED_KEYS
-      reserved_keys << :lit_default_copy
+      reserved_keys = I18n.const_get(:RESERVED_KEYS) + %i[lit_default_copy]
       I18n.send(:remove_const, :RESERVED_KEYS)
-      I18n.const_set(:RESERVED_KEYS, reserved_keys)
+      I18n.const_set(:RESERVED_KEYS, reserved_keys.freeze)
     end
 
     def translate(locale, key, options = {})
@@ -59,7 +59,8 @@ module Lit
       return true if options[:default].is_a?(String)
       return true if options[:default].is_a?(Array) && \
                      (options[:default].first.is_a?(String) || \
-                      options[:default].first.is_a?(Symbol))
+                      options[:default].first.is_a?(Symbol) || \
+                      options[:default].first.is_a?(Array))
       false
     end
 
@@ -69,11 +70,11 @@ module Lit
       parts = I18n.normalize_keys(locale, key, scope, options[:separator])
       key_with_locale = parts.join('.')
 
-      ## check in cache or in simple backend
+      # check in cache or in simple backend
       content = @cache[key_with_locale] || super
       return content if parts.size <= 1
 
-      if content.nil? && should_cache?(key_with_locale)
+      if content.nil? && should_cache?(key_with_locale, options)
         new_content = @cache.init_key_with_value(key_with_locale, content)
         content = new_content if content.nil? # Content can change when Lit.humanize is true for example
         # so there is no content in cache - it might not be if ie. we're doing
@@ -116,7 +117,7 @@ module Lit
           end
         end
       end
-      ## return translated content
+      # return translated content
       content
     end
 
@@ -180,8 +181,10 @@ module Lit
       Lit.ignored_keys.any?{ |k| key_without_locale.start_with?(k) }
     end
 
-    def should_cache?(key_with_locale)
-      return false if @cache.has_key?(key_with_locale)
+    def should_cache?(key_with_locale, options)
+      if @cache.has_key?(key_with_locale)
+        return false unless options[:default]
+      end
 
       _, key_without_locale = ::Lit::Cache.split_key(key_with_locale)
       return false if is_ignored_key(key_without_locale)

data/lib/lit/import.rb

@@ -0,0 +1,179 @@
+require 'csv'
+
+module Lit
+  class Import
+    class << self
+      def call(*args)
+        new(*args).perform
+      end
+    end
+
+    attr_reader :input, :locale_keys, :format, :skip_nil
+
+    def initialize(input:, locale_keys: [], format:, skip_nil: true, dry_run: false, raw: false)
+      raise ArgumentError, 'format must be yaml or csv' if %i[yaml csv].exclude?(format.to_sym)
+      @input = input
+      @locale_keys = locale_keys.presence || I18n.available_locales
+      @format = format
+      @skip_nil = skip_nil
+      @dry_run = dry_run
+      @raw = raw
+    end
+
+    def perform
+      send(:"import_#{format}")
+    end
+
+    private
+
+    def import_yaml
+      validate_yaml
+      locale_keys.each do |locale|
+        I18n.with_locale(locale) do
+          yml = parsed_yaml[locale.to_s]
+          Hash[*Lit::Cache.flatten_hash(yml)].each do |key, default_translation|
+            next if default_translation.nil? && skip_nil
+            puts key
+            upsert(locale, key, default_translation)
+          end
+        end
+      end
+    rescue Psych::SyntaxError => e
+      raise ArgumentError, "Invalid YAML file: #{e.message}", cause: e
+    end
+
+    def import_csv
+      validate_csv
+      processed_csv = preprocess_csv
+
+      processed_csv.each do |row|
+        key = row.first
+        row_translations = Hash[locales_in_csv.zip(row.drop(1))]
+        row_translations.each do |locale, value|
+          next unless locale_keys.blank? || locale_keys.map(&:to_sym).include?(locale.to_sym)
+          next if value.nil? && skip_nil
+          puts key
+          upsert(locale, key, value)
+        end
+      end
+    rescue CSV::MalformedCSVError => e
+      raise ArgumentError, "Invalid CSV file: #{e.message}", cause: e
+    end
+
+    def validate_yaml
+      errors = []
+
+      # YAML.load can return false, hence not using #empty?
+      errors << :yaml_is_empty if parsed_yaml.blank?
+
+      if parsed_yaml.present? &&
+         (locale_keys.map(&:to_sym) - parsed_yaml.keys.map(&:to_sym)).any?
+        errors << :not_all_requested_locales_included_in_header
+      end
+
+      fail ArgumentError, errors.map { |e| e.to_s.humanize }.to_sentence if errors.any?
+    end
+
+    def validate_csv # rubocop:disable Metrics/AbcSize
+      errors = []
+
+      # CSV may not be empty
+      errors << :csv_is_empty if parsed_csv.empty?
+
+      # verify CSV header
+      if !parsed_csv.empty? &&
+         (locale_keys.map(&:to_s) - parsed_csv[0].drop(1)).any?
+        errors << :not_all_requested_locales_included_in_header
+      end
+
+      # any further checks that we at some time think of should fall here
+
+      fail ArgumentError, errors.map { |e| e.to_s.humanize }.to_sentence if errors.any?
+    end
+
+    # the main task of this routine is to replace blanks with nils (in CSV it cannot be distinguished,
+    # so in order for :skip_nil option to work as intended blanks must be treated as nil);
+    # as well as that, we need to look for multiple occurrences of certain keys and merge them
+    # into arrays
+    def preprocess_csv
+      concatenate_arrays(replace_blanks(parsed_csv))
+    end
+
+    def parsed_csv
+      @parsed_csv ||=
+        begin
+          CSV.parse(input)
+        rescue CSV::MalformedCSVError
+          # Some Excel versions tend to save CSVs with columns separated with tabs instead
+          # of commas. Let's try that out if needed.
+          CSV.parse(input, col_sep: "\t")
+        end
+    end
+
+    def parsed_yaml
+      @parsed_yaml ||= YAML.load(input)
+    end
+
+    def locales_in_csv
+      @locales_in_csv ||= parsed_csv.first.drop(1)
+    end
+
+    # This is mean to insert a value for a key in a given locale
+    # using some kind of strategy which depends on the service's options.
+    #
+    # For instance, when @raw option is true (it's the default),
+    # if a key already exists, it overrides the default_value of the
+    # existing localization key; otherwise, with @raw set to false,
+    # it keeps the default as it is and, no matter if a translated value
+    # is there, translated_value is overridden with the imported one
+    # and is_changed is set to true.
+    def upsert(locale, key, value) # rubocop:disable Metrics/MethodLength
+      I18n.with_locale(locale) do
+        # when an array has to be inserted with a default value, it needs to
+        # be done like:
+        # I18n.t('foo', default: [['bar', 'baz']])
+        # because without the double array, array items are treated as fallback keys
+        # - then, the last array element is the final fallback; so in this case we
+        # don't specify fallback keys and only specify the final fallback, which
+        # is the array
+        val = value.is_a?(Array) ? [value] : value
+        I18n.t(key, default: val)
+        unless @raw
+          # this indicates that this translation already exists
+          existing_translation =
+            Lit::Localization.joins(:locale, :localization_key)
+                             .find_by('localization_key = ? and locale = ?',
+                                      key, locale)
+          if existing_translation
+            existing_translation.update(translated_value: value, is_changed: true)
+            lkey = existing_translation.localization_key
+            lkey.update(is_deleted: false) if lkey.is_deleted
+          end
+        end
+      end
+    end
+
+    def concatenate_arrays(csv) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/LineLength
+      csv.inject([]) do |accu, row|
+        if row.first == accu.last&.first # equal keys
+          accu.tap do
+            accu[-1] = [
+              row.first,
+              *accu[-1].drop(1)
+                       .map { |x| Array.wrap(x) }
+                       .zip(row.drop(1)).map(&:flatten)
+            ]
+          end
+        else
+          accu << row
+        end
+      end
+    end
+
+    def replace_blanks(csv)
+      csv.drop(1).each do |row|
+        row.replace(row.map(&:presence))
+      end
+    end
+  end
+end