i18n-processes 0.1.0

data/lib/i18n/processes/data/file_formats.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'yaml'
+module I18n
+  module Processes
+    module Data
+      module FileFormats
+        def self.included(base)
+          base.extend ClassMethods
+        end
+
+        def adapter_dump(tree, format)
+          adapter_op :dump, format, tree, write_config(format)
+        end
+
+        # @return [Hash]
+        def adapter_parse(tree, format)
+          adapter_op :parse, format, tree, read_config(format)
+        end
+
+        def adapter_op(op, format, tree, config)
+          self.class.adapter_by_name(format).send(op, tree, config)
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          raise CommandError, "#{format} #{op} error: #{e.message}"
+        end
+
+
+
+        protected
+
+        def write_config(format)
+          (config[format] || {})[:write]
+        end
+
+        def read_config(format)
+          (config[format] || {})[:read]
+        end
+
+
+
+        # def source_config(format)
+        #   (config[format] || {})[:source]
+        # end
+        #
+        # def translation_config(format)
+        #   (config[format] || {})[:translation]
+        # end
+        #
+        # def translated_config(format)
+        #   (config[format] || {})[:translated]
+        # end
+
+        # @return [Hash]
+        def load_file(path)
+          adapter_parse read_file(path), self.class.adapter_name_for_path(path)
+        end
+
+        # @return [String]
+        def read_file(path)
+          ::File.read(path, encoding: 'UTF-8')
+        end
+
+        def write_tree(path, tree)
+          hash = tree.to_hash(true)
+          adapter = self.class.adapter_name_for_path(path)
+          content = adapter_dump(hash, adapter)
+          # Ignore unchanged data
+          return if File.file?(path) && content == read_file(path)
+          ::FileUtils.mkpath(File.dirname(path))
+          ::File.open(path, 'w') { |f| f.write content }
+        end
+
+        def normalized?(path, tree)
+          return false unless File.file?(path)
+          read_file(path) == adapter_dump(tree.to_hash(true), self.class.adapter_name_for_path(path))
+        end
+
+        module ClassMethods
+          # @param pattern [String] File.fnmatch pattern
+          # @param adapter [responds to parse(string)->hash and dump(hash)->string]
+          def register_adapter(name, pattern, adapter)
+            (@fn_patterns ||= []) << [name, pattern, adapter]
+          end
+
+          def adapter_name_for_path(path)
+            @fn_patterns.detect do |(_name, pattern, _adapter)|
+              ::File.fnmatch(pattern, path)
+            end.try(:first) || fail(
+              CommandError, "Adapter not found for #{path}. Registered adapters: #{@fn_patterns.inspect}"
+            )
+          end
+
+          def adapter_names
+            @fn_patterns.map(&:first)
+          end
+
+          def adapter_by_name(name)
+            name = name.to_s
+            @fn_patterns.detect do |(adapter_name, _pattern, _adapter)|
+              adapter_name.to_s == name
+            end.try(:last) || fail(
+              CommandError,
+              "Adapter with name #{name.inspect} not found. Registered adapters: #{@fn_patterns.inspect}"
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/i18n/processes/data/file_system.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/data/file_system_base'
+require 'i18n/processes/data/adapter/json_adapter'
+require 'i18n/processes/data/adapter/yaml_adapter'
+
+module I18n::Processes
+  module Data
+    class FileSystem < FileSystemBase
+      register_adapter :yaml, '*.yml', Adapter::YamlAdapter
+      register_adapter :json, '*.json', Adapter::JsonAdapter
+    end
+  end
+end

data/lib/i18n/processes/data/file_system_base.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/data/tree/node'
+require 'i18n/processes/data/router/pattern_router'
+require 'i18n/processes/data/router/conservative_router'
+require 'i18n/processes/data/file_formats'
+require 'i18n/processes/key_pattern_matching'
+
+module I18n::Processes
+  module Data
+    class FileSystemBase # rubocop:disable Metrics/ClassLength
+      include I18n::Processes::Data::FileFormats
+      include I18n::Processes::Logging
+
+      attr_reader :config, :base_locale, :locales
+      attr_writer :locales
+
+      DEFAULTS = {
+        read:  ['config/locales/%{locale}.yml'],
+        write: ['config/locales/%{locale}.yml']
+      }.freeze
+
+      def initialize(config = {})
+        self.config  = config.except(:base_locale, :locales)
+        @base_locale = config[:base_locale]
+        locales = config[:locales].presence
+        @locales = LocaleList.normalize_locale_list(locales || available_locales, base_locale, true)
+        if locales.present?
+          log_verbose "locales read from config #{@locales * ', '}"
+        else
+          log_verbose "locales inferred from data: #{@locales * ', '}"
+        end
+      end
+
+      # @param [String, Symbol] locale
+      # @return [::I18n::Processes::Data::Siblings]
+      def get(locale)
+        locale = locale.to_s
+        @trees ||= {}
+        @trees[locale] ||= read_locale(locale)
+      end
+
+      alias [] get
+
+      # @param [String, Symbol] locale
+      # @return [::I18n::Processes::Data::Siblings]
+      def external(locale)
+        locale = locale.to_s
+        @external ||= {}
+        @external[locale] ||= read_locale(locale, paths: config[:external])
+      end
+
+      # set locale tree
+      # @param [String, Symbol] locale
+      # @param [::I18n::Processes::Data::Siblings] tree
+      def set(locale, tree)
+        locale = locale.to_s
+        @trees.delete(locale) if @trees
+        paths_before = Set.new(get(locale)[locale].leaves.map { |node| node.data[:path] })
+        paths_after = Set.new([])
+        # $stderr.puts Rainbow("locale: #{locale}").green
+        # $stderr.puts Rainbow("tree: #{tree.class}").green
+        router.route locale, tree do |path, tree_slice|
+          paths_after << path
+          write_tree path, tree_slice
+        end
+        (paths_before - paths_after).each do |path|
+          FileUtils.remove_file(path) if File.exist?(path)
+        end
+        @trees.delete(locale) if @trees
+        @available_locales = nil
+      end
+
+      alias []= set
+
+      # @param [String] locale
+      # @return [Array<String>] paths to files that are not normalized
+      def non_normalized_paths(locale)
+        router.route(locale, get(locale))
+              .reject { |path, tree_slice| normalized?(path, tree_slice) }
+              .map(&:first)
+      end
+
+      def write(forest)
+        forest.each { |root| set(root.key, root.to_siblings) }
+      end
+
+      def merge!(forest)
+        ## forest is a sibling
+        forest.inject(Tree::Siblings.new) do |result, root|
+          # root is a node, merge is a sibling
+          locale = root.key
+          merged = get(locale).merge(root)
+          $stderr.puts Rainbow("root: #{root}").green
+          $stderr.puts Rainbow("locale: #{locale}").green
+          $stderr.puts Rainbow("merged: #{merged}").green
+          set locale, merged
+          result.merge! merged
+        end
+      end
+
+      def remove_by_key!(forest)
+        forest.inject(Tree::Siblings.new) do |removed, root|
+          locale = root.key
+          locale_data = get(locale)
+          subtracted = locale_data.subtract_by_key(forest)
+          set locale, subtracted
+          removed.merge! locale_data.subtract_by_key(subtracted)
+        end
+      end
+
+      # @return self
+      def reload
+        @trees             = nil
+        @available_locales = nil
+        self
+      end
+
+      # Get available locales from the list of file names to read
+      def available_locales
+        @available_locales ||= begin
+          locales = Set.new
+          Array(config[:read]).map do |pattern|
+            [pattern, Dir.glob(format(pattern, locale: '*'))] if pattern.include?('%{locale}')
+          end.compact.each do |pattern, paths|
+            p  = pattern.gsub('\\', '\\\\').gsub('/', '\/').gsub('.', '\.')
+            p  = p.gsub(/(\*+)/) { Regexp.last_match(1) == '**' ? '.*' : '[^/]*?' }.gsub('%{locale}', '([^/.]+)')
+            re = /\A#{p}\z/
+            paths.each do |path|
+              locales << Regexp.last_match(1) if re =~ path
+            end
+          end
+          locales
+        end
+      end
+
+      def t(key, locale)
+        tree = self[locale.to_s]
+        return unless tree
+        tree[locale][key].try(:value_or_children_hash)
+      end
+
+      def config=(config)
+        @config = DEFAULTS.deep_merge((config || {}).reject { |_k, v| v.nil? })
+        reload
+      end
+
+      def with_router(router)
+        router_was  = self.router
+        self.router = router
+        yield
+      ensure
+        self.router = router_was
+      end
+
+      ROUTER_NAME_ALIASES = {
+        'conservative_router' => 'I18n::Processes::Data::Router::ConservativeRouter',
+        'pattern_router' => 'I18n::Processes::Data::Router::PatternRouter'
+      }.freeze
+      def router
+        @router ||= begin
+          name = @config[:router].presence || 'conservative_router'
+          name = ROUTER_NAME_ALIASES[name] || name
+          router_class = ActiveSupport::Inflector.constantize(name)
+          router_class.new(self, @config.merge(base_locale: base_locale, locales: locales))
+        end
+      end
+      attr_writer :router
+
+      protected
+
+      def read_locale(locale, paths: config[:read])
+        Array(paths).flat_map do |path|
+          Dir.glob format(path, locale: locale)
+        end.map do |path|
+          [path.freeze, load_file(path) || {}]
+        end.map do |path, data|
+          filter_nil_keys! path, data
+          Data::Tree::Siblings.from_nested_hash(data).tap do |s|
+            s.leaves { |x| x.data.update(path: path, locale: locale) }
+          end
+        end.reduce(Tree::Siblings[locale => {}], :merge!)
+      end
+
+      def filter_nil_keys!(path, data, suffix = [])
+        data.each do |key, value|
+          if key.nil?
+            data.delete(key)
+            log_warn <<-TEXT
+Skipping a nil key found in #{path.inspect}:
+  key: #{suffix.join('.')}.`nil`
+  value: #{value.inspect}
+Nil keys are not supported by i18n.
+The following unquoted YAML keys result in a nil key:
+  #{%w[null Null NULL ~].join(', ')}
+See http://yaml.org/type/null.html
+TEXT
+          elsif value.is_a?(Hash)
+            filter_nil_keys! path, value, suffix + [key]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/i18n/processes/data/router/conservative_router.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/data/router/pattern_router'
+
+module I18n::Processes
+  module Data::Router
+    # Keep the path, or infer from base locale
+    class ConservativeRouter < PatternRouter
+      def initialize(adapter, config)
+        @adapter     = adapter
+        @base_locale = config[:base_locale]
+        @locales     = config[:locales]
+        super
+      end
+
+      def route(locale, forest, &block)
+        return to_enum(:route, locale, forest) unless block
+        out = Hash.new { |hash, key| hash[key] = Set.new }
+        not_found = Set.new
+        forest.keys do |key, _node|
+          path = key_path(locale, key)
+          $stderr.puts Rainbow("path in route: #{path}").green
+          # infer from another locale
+          unless path
+            inferred_from = (locales - [locale]).detect { |loc| path = key_path(loc, key) }
+            path = LocalePathname.replace_locale(path, inferred_from, locale) if inferred_from
+          end
+          key_with_locale = "#{locale}.#{key}"
+          $stderr.puts Rainbow("key_with_locale in route: #{key_with_locale}").green
+          if path
+            out[path] << key_with_locale
+          else
+            not_found << key_with_locale
+          end
+        end
+
+
+        if not_found.present?
+          # fall back to pattern router
+          not_found_tree = forest.select_keys(root: true) { |key, _| not_found.include?(key) }
+          super(locale, not_found_tree) do |path, tree|
+            out[path] += tree.key_names(root: true)
+          end
+        end
+
+        out.each do |dest, keys|
+          # $stderr.puts Rainbow("dest in route: #{dest}").green
+          # $stderr.puts Rainbow("dest in route: #{dest}").green
+          block.yield dest, forest.select_keys(root: true) { |key, _| keys.include?(key) }
+        end
+      end
+
+      protected
+
+      def base_tree
+        adapter[base_locale]
+      end
+
+      def key_path(locale, key)
+        adapter[locale]["#{locale}.#{key}"].try(:data).try(:[], :path)
+      end
+
+      attr_reader :adapter, :base_locale, :locales
+    end
+  end
+end

data/lib/i18n/processes/data/router/pattern_router.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/key_pattern_matching'
+require 'i18n/processes/data/tree/node'
+
+module I18n::Processes
+  module Data::Router
+    # Route based on key name
+    class PatternRouter
+      include ::I18n::Processes::KeyPatternMatching
+
+      attr_reader :routes
+      # @option data_config write [Array] of routes
+      # @example
+      #   {write:
+      #     # keys matched top to bottom
+      #     [['devise.*', 'config/locales/devise.%{locale}.yml'],
+      #     # default catch-all (same as ['*', 'config/locales/%{locale}.yml'])
+      #     'config/locales/%{locale}.yml']}
+      def initialize(_adapter, data_config)
+        @routes_config = data_config[:write]
+        @routes = compile_routes @routes_config
+      end
+
+      # Route keys to destinations
+      # @param forest [I18n::Processes::Data::Tree::Siblings] forest roots are locales.
+      # @yieldparam [String] dest_path
+      # @yieldparam [I18n::Processes::Data::Tree::Siblings] tree_slice
+      # @return [Hash] mapping of destination => [ [key, value], ... ]
+      def route(locale, forest, &block)
+        return to_enum(:route, locale, forest) unless block
+        locale = locale.to_s
+        out = {}
+        forest.keys do |key, _node|
+          pattern, path = routes.detect { |route| route[0] =~ key }
+          if pattern
+            key_match = $~
+            path      = format(path, locale: locale)
+            path.gsub!(/\\\d+/) { |m| key_match[m[1..-1].to_i] }
+            (out[path] ||= Set.new) << "#{locale}.#{key}"
+          else
+            fail CommandError, "Cannot route key #{key}. Routes are #{@routes_config.inspect}"
+          end
+        end
+        out.each do |dest, keys|
+          block.yield dest,
+                      forest.select_keys(root: true) { |key, _| keys.include?(key) }
+        end
+      end
+
+      private
+
+      def compile_routes(routes)
+        routes.map { |x| x.is_a?(String) ? ['*', x] : x }.map do |x|
+          [compile_key_pattern(x[0]), x[1]]
+        end
+      end
+    end
+  end
+end

data/lib/i18n/processes/data/tree/node.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require 'i18n/processes/data/tree/traversal'
+require 'i18n/processes/data/tree/siblings'
+require 'i18n/processes/rainbow_utils'
+module I18n::Processes::Data::Tree
+  class Node # rubocop:disable Metrics/ClassLength
+    include Enumerable
+    include Traversal
+
+    attr_accessor :value
+    attr_reader :key, :children, :parent
+
+    # rubocop:disable Metrics/ParameterLists
+    def initialize(key:, value: nil, data: nil, parent: nil, children: nil, warn_about_add_children_to_leaf: true)
+      @key = key
+      @key = @key.to_s.freeze if @key
+      @value = value
+      @data = data
+      @parent = parent
+      @warn_about_add_children_to_leaf = warn_about_add_children_to_leaf
+      self.children = (children if children)
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def attributes
+      { key: @key, value: @value, data: @data.try(:clone), parent: @parent, children: @children }
+    end
+
+    def derive(new_attr = {})
+      self.class.new(attributes.merge(new_attr))
+    end
+
+    def children=(children)
+      @children = case children
+                  when Siblings
+                    children.parent == self ? children : children.derive(parent: self)
+                  when NilClass
+                    nil
+                  else
+                    Siblings.new(
+                      nodes: children,
+                      parent: self,
+                      warn_about_add_children_to_leaf: @warn_about_add_children_to_leaf
+                    )
+                  end
+      dirty!
+    end
+
+    def each(&block)
+      return to_enum(:each) { 1 } unless block
+      block.yield(self)
+      self
+    end
+
+    def value_or_children_hash
+      leaf? ? value : children.try(:to_hash)
+    end
+
+    def leaf?
+      !children
+    end
+
+    # a node with key nil is considered Empty. this is to allow for using these nodes instead of nils
+    def root?
+      !parent?
+    end
+
+    def parent?
+      !parent.nil?
+    end
+
+    def children?
+      children && !children.empty?
+    end
+
+    def data
+      @data ||= {}
+    end
+
+    def data?
+      @data.present?
+    end
+
+    def reference?
+      value.is_a?(Symbol)
+    end
+
+    def get(key)
+      children.get(key)
+    end
+
+    alias [] get
+
+    # append and reparent nodes
+    def append!(nodes)
+      if @children
+        @children.merge!(nodes)
+      else
+        @children = Siblings.new(nodes: nodes, parent: self)
+      end
+      self
+    end
+
+    def append(nodes)
+      derive.append!(nodes)
+    end
+
+    def full_key(root: true)
+      @full_key ||= {}
+      @full_key[root] ||= "#{"#{parent.full_key(root: root)}." if parent? && (root || parent.parent?)}#{key}"
+    end
+
+    def walk_to_root(&visitor)
+      return to_enum(:walk_to_root) unless visitor
+      visitor.yield self
+      parent.walk_to_root(&visitor) if parent?
+    end
+
+    def root
+      p = nil
+      walk_to_root { |node| p = node }
+      p
+    end
+
+    def walk_from_root(&visitor)
+      return to_enum(:walk_from_root) unless visitor
+      walk_to_root.reverse_each do |node|
+        visitor.yield node
+      end
+    end
+
+    def set(full_key, node)
+      (@children ||= Siblings.new(parent: self)).set(full_key, node)
+      dirty!
+      node
+    end
+
+    alias []= set
+
+    def to_nodes
+      Nodes.new([self])
+    end
+
+    def to_siblings
+      parent && parent.children || Siblings.new(nodes: [self])
+    end
+
+    def to_hash(sort = false)
+      (@hash ||= {})[sort] ||= begin
+        children_hash = children ? children.to_hash(sort) : {}
+        if key.nil?
+          children_hash
+        elsif leaf?
+          { key => value }
+        else
+          { key => children_hash }
+        end
+      end
+    end
+
+    delegate :to_json, to: :to_hash
+    delegate :to_yaml, to: :to_hash
+
+    def inspect(level = 0)
+      label = if key.nil?
+                I18n::Processes::RainbowUtils.faint_color('∅')
+              else
+                [Rainbow(key).color(1 + level % 15),
+                 (": #{format_value_for_inspect(value)}" if leaf?),
+                 (" #{data}" if data?)].compact.join
+              end
+      ['  ' * level, label, ("\n" + children.map { |c| c.inspect(level + 1) }.join("\n") if children?)].compact.join
+    end
+
+    def format_value_for_inspect(value)
+      if value.is_a?(Symbol)
+        "#{Rainbow('⮕ ').bright.yellow}#{Rainbow(value).yellow}"
+      else
+        Rainbow(value).cyan
+      end
+    end
+
+    protected
+
+    def dirty!
+      @hash = nil
+      @full_key = nil
+    end
+
+    class << self
+      # value can be a nested hash
+      def from_key_value(key, value)
+        Node.new(key: key.try(:to_s)).tap do |node|
+          if value.is_a?(Hash)
+            node.children = Siblings.from_nested_hash(value)
+          else
+            node.value = value
+          end
+        end
+      end
+    end
+  end
+end