translatomatic 0.1.3 → 0.2.0

data/lib/translatomatic/provider/my_memory.rb

@@ -0,0 +1,75 @@
+module Translatomatic
+  module Provider
+    # Interface to the MyMemory translation API
+    # @see https://mymemory.translated.net/doc/
+    class MyMemory < Base
+      define_option :mymemory_api_key, use_env: true,
+                                       desc: t('provider.mymemory.api_key')
+      define_option :mymemory_email, use_env: true,
+                                     desc: t('provider.email_address')
+
+      # Create a new MyMemory provider instance
+      def initialize(options = {})
+        super(options)
+        @key = options[:mymemory_api_key] || ENV['MYMEMORY_API_KEY']
+        @email = options[:mymemory_email] || ENV['MYMEMORY_EMAIL']
+        @query_options = {}
+        @query_options[:de] = @email if @email
+        @query_options.merge!(key: @key) if @key
+      end
+
+      # (see Base#languages)
+      def languages
+        Locale.language_codes
+      end
+
+      # Upload a set of translations to MyMemory
+      # @param tmx [Translatomatic::TMX::Document] TMX document
+      # @return [void]
+      def upload(tmx)
+        body = [
+          { key: :tmx, filename: 'import.tmx',
+            content: tmx.to_xml, mime_type: 'application/xml' },
+          { key: :private, value: 0 }
+        ]
+        response = http_client.post(UPLOAD_URL, body)
+        log.debug(t('provider.share_response',
+                    response: response.body.inspect))
+      end
+
+      private
+
+      GET_URL = 'https://api.mymemory.translated.net/get'.freeze
+      UPLOAD_URL = 'https://api.mymemory.translated.net/tmx/import'.freeze
+      MAIN_URL = 'https://mymemory.api.net'.freeze
+
+      def perform_translate(strings, from, to)
+        perform_fetch_translations(GET_URL, strings, from, to)
+      end
+
+      def fetch_translations(string, from, to)
+        response = http_client.get(GET_URL, {
+          langpair: from.to_s + '|' + to.to_s,
+          q: string # multiple q strings not supported (tested 20180101)
+        }.merge(@query_options))
+
+        log.debug("#{name} response: #{response.body}")
+        data = JSON.parse(response.body)
+        # matches = data['matches'] # all translations
+        # matches.collect { |i| match_data(i) }
+        result = try_hash(data, 'responseData', 'translatedText')
+        add_translations(string, result)
+      end
+
+      # https://mymemory.translated.net/doc/features.php
+      def match_data(match)
+        {
+          translation: match['translation'],
+          quality: match['quality'],
+          usage_count: match['usage-count'],
+          match: match['match'], # partial matches, see features.php above
+        }
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/yandex.rb

@@ -0,0 +1,61 @@
+module Translatomatic
+  module Provider
+    # Interface to the Yandex translation API
+    # @see https://tech.yandex.com/translate/
+    class Yandex < Base
+      define_option :yandex_api_key, use_env: true,
+                                     desc: t('provider.yandex.api_key')
+
+      # Create a new Yandex provider instance
+      def initialize(options = {})
+        super(options)
+        @api_key = options[:yandex_api_key] || ENV['YANDEX_API_KEY']
+        raise t('provider.yandex.key_required') if @api_key.nil?
+      end
+
+      # (see Base#languages)
+      def languages
+        @languages ||= begin
+          response = http_client.post(LANGUAGES_URL, key: @api_key, ui: 'en')
+          data = JSON.parse(response.body) || {}
+          langs = data['langs'] || {}
+          langs.keys.flatten.uniq
+        end
+      end
+
+      private
+
+      BASE_URL = 'https://translate.yandex.net/api/v1.5/tr.json'.freeze
+      TRANSLATE_URL = (BASE_URL + '/translate').freeze
+      LANGUAGES_URL = (BASE_URL + '/getLangs').freeze
+      LIMIT = [nil, 10_000].freeze # strings, characters per request
+
+      def perform_translate(strings, from, to)
+        batcher(strings, max_count: LIMIT[0], max_length: LIMIT[1])
+          .each_batch do |texts|
+          fetch_translations(texts, from, to)
+        end
+      end
+
+      def fetch_translations(strings, from, to)
+        body = request_body(strings, from, to)
+        response = http_client.post(TRANSLATE_URL, body)
+        log.debug("#{name} response: #{response.body}")
+        data = JSON.parse(response.body)
+        result = data['text'] || []
+        strings.zip(result).each do |original, translated|
+          add_translations(original, translated)
+        end
+      end
+
+      def request_body(strings, from, to)
+        {
+          key: @api_key,
+          text: strings,
+          lang: from.language + '-' + to.language,
+          format: 'plain' # 'html'
+        }
+      end
+    end
+  end
+end

data/lib/translatomatic/resource_file.rb

@@ -4,76 +4,81 @@ module Translatomatic
   module ResourceFile
     class << self
       include Translatomatic::Util
-    end
 
-    # Load a resource file. If locale is not specified, the locale of the
-    # file will be determined from the filename, or else the current default
-    # locale will be used.
-    # @param path [String] Path to the resource file
-    # @param locale [String] Locale of the resource file
-    # @return [Translatomatic::ResourceFile::Base] The resource file, or nil
-    #   if the file type is unsupported.
-    def self.load(path, locale = nil)
-      path = path.kind_of?(Pathname) ? path : Pathname.new(path)
-      types_for_path(path).each do |klass|
-        log.debug(t("file.loading", file: path, name: klass.name.demodulize))
-        return klass.new(path, locale: locale)
+      # Load a resource file. If options[:locale] is not specified,
+      # the locale of the file will be determined from the filename,
+      # or else the current default locale will be used.
+      # @param path [String] Path to the resource file
+      # @return [Translatomatic::ResourceFile::Base] The resource file, or nil
+      #   if the file type is unsupported.
+      def load(path, options = {})
+        path = path.is_a?(Pathname) ? path : Pathname.new(path)
+        types_for_path(path).each do |klass|
+          log.debug(t('file.loading', file: path, name: klass.name.demodulize))
+          return klass.new(path, options)
+        end
+        nil
       end
-      nil
-    end
 
-    # Create a new resource file
-    def self.create(path, locale = nil)
-      klass = const_get(klass_name)
-      klass.new
-    end
+      # Create a new resource file
+      def create(path, options = {})
+        klass = types_for_path(path).first
+        return nil unless klass
+        file = klass.new
+        file.path = path
+        file.locale = build_locale(options[:locale])
+        file
+      end
 
-    # Find all resource files under the given directory. Follows symlinks.
-    # @param path [String, Pathname] The path to search from
-    # @return [Array<Translatomatic::ResourceFile>] Resource files found
-    def self.find(path, options = {})
-      files = []
-      include_dot_directories = options[:include_dot_directories]
-      path = Pathname.new(path) unless path.kind_of?(Pathname)
-      path.find do |file|
-        if !include_dot_directories && file.basename.to_s[0] == ?.
-          Find.prune
-        else
-          resource = load(file)
-          files << resource if resource
+      # Find all resource files under the given directory. Follows symlinks.
+      # @param path [String, Pathname] The path to search from
+      # @return [Array<Translatomatic::ResourceFile>] Resource files found
+      def find(path, options = {})
+        files = []
+        include_dot_directories = options[:include_dot_directories]
+        path = Pathname.new(path) unless path.is_a?(Pathname)
+        path.find do |file|
+          puts "loading #{file}"
+          if !include_dot_directories && file.basename.to_s[0] == '.'
+            Find.prune
+          else
+            resource = load(file)
+            files << resource if resource
+          end
         end
+        files
       end
-      files
-    end
 
-    # Find all configured resource file classes
-    # @return [Array<Class>] Available resource file classes
-    def self.types
-      @types ||= self.constants.map { |c| self.const_get(c) }.select do |klass|
-        klass.is_a?(Class) && klass != Base
+      # Find all configured resource file classes
+      # @return [Array<Class>] Available resource file classes
+      def types
+        @types ||= constants.map { |c| const_get(c) }.select do |klass|
+          klass.is_a?(Class) && klass != Base && klass < Base
+        end
       end
-    end
 
-    private
+      private
 
-    # find classes that can load the given path by file extension
-    def self.types_for_path(path)
-      path = path.kind_of?(Pathname) ? path : Pathname.new(path)
-      types.select { |klass| klass.enabled? && extension_match(klass, path) }
-    end
+      # find classes that can load the given path by file extension
+      def types_for_path(path)
+        path = path.is_a?(Pathname) ? path : Pathname.new(path)
+        types.select { |klass| klass.enabled? && extension_match(klass, path) }
+      end
 
-    def self.extension_match(klass, path)
-      filename = path.basename.to_s.downcase
-      klass.extensions.each do |extension|
-        # don't match end of line in case file has locale extension
-        return true if filename.match(/\.#{extension}/)
+      def extension_match(klass, path)
+        filename = path.basename.to_s.downcase
+        klass.extensions.each do |extension|
+          # don't match end of line in case file has locale extension
+          return true if filename =~ /\.#{extension}/
+        end
+        false
       end
-      false
     end
   end
 end
 
 require 'translatomatic/resource_file/base'
+require 'translatomatic/resource_file/key_value_support'
 require 'translatomatic/resource_file/yaml'
 require 'translatomatic/resource_file/properties'
 require 'translatomatic/resource_file/text'
@@ -85,3 +90,4 @@ require 'translatomatic/resource_file/plist'
 require 'translatomatic/resource_file/resw'
 require 'translatomatic/resource_file/subtitle'
 require 'translatomatic/resource_file/csv'
+require 'translatomatic/resource_file/po'

data/lib/translatomatic/resource_file/base.rb

@@ -1,269 +1,203 @@
-# Base class for resource file implementations
-# @abstract Subclasses implement different types of resource files
-class Translatomatic::ResourceFile::Base
-
-  attr_accessor :locale
-  attr_accessor :path
-
-  # @return [Hash<String,String>] key -> value properties
-  attr_reader :properties
-
-  # @return [boolean] True if this resource format is enabled
-  def self.enabled?
-    true
-  end
-
-  # @return [Array<String>] File extensions supported by this resource file
-  def self.extensions
-    raise "extensions must be implemented by subclass"
-  end
-
-  # @return [boolean] True if the file format consists of keys and values
-  def self.is_key_value?
-    false
-  end
-
-  # @return [boolean] true if this resource file supports variable interpolation
-  def self.supports_variable_interpolation?
-    false
-  end
-
-  # Create a new resource file or load an existing file.
-  # If options[:locale] is unspecified, attempts to determine
-  # the locale of the file automatically, and if that fails,
-  # uses the default locale.
-  # Raises an exception if errors were encountered loading the file.
-  # @param path [String] Path to the file
-  # @param options [Hash<Symbol,String>] Options
-  # @return [Translatomatic::ResourceFile::Base] the resource file.
-  def initialize(path = nil, options = {})
-    raise "expected options hash" if options && !options.kind_of?(Hash)
-    raise t("file.unsupported", file: path) unless self.class.enabled?
-    @options = options || {}
-    @properties = {}
-    @path = path.nil? || path.kind_of?(Pathname) ? path : Pathname.new(path)
-    update_locale
-    init
-    load if @path && @path.exist?
-  end
-
-  # Save the resource file.
-  # @param target [Pathname] The destination path
-  # @param options [Hash<Symbol, Object>] Output format options
-  # @return [void]
-  def save(target = path, options = {})
-    raise "save(path) must be implemented by subclass"
-  end
-
-  # @return [String] The format of this resource file, e.g. "Properties"
-  def format
-    self.class.name.demodulize.downcase.to_sym
-  end
-
-  # Create a path for the current resource file with a given locale
-  # @param locale [String] The target locale
-  # @return [Pathname] The path of this resource file modified for the given locale
-  def locale_path(locale)
-    basename = path.sub_ext('').basename.to_s
-
-    extlist = extension_list
-    if extlist.length >= 2 && loc_idx = find_locale(extlist)
-      # extension(s) contains locale, replace it
-      extlist[loc_idx] = locale.to_s
-    elsif valid_locale?(basename)
-      # basename is a locale name, replace it
-      path.dirname + (locale.to_s + path.extname)
-    else
-      # remove any underscore and trailing text from basename
-      deunderscored = basename.sub(/_.*?$/, '')
-      # add _locale.ext
-      filename = deunderscored + "_" + locale.to_s + path.extname
-      path.dirname + filename
-    end
-  end
-
-  # Set all properties
-  # @param properties [Hash<String,String>] New properties
-  def properties=(properties)
-    # use set rather that set @properties directly as subclasses override set()
-    properties.each do |key, value|
-      set(key, value)
-    end
-  end
+module Translatomatic
+  module ResourceFile
+    # Base class for resource file implementations
+    # @abstract Subclasses implement different types of resource files
+    class Base
+      # @return [Hash<Symbol,Object] Options used in the constructor
+      attr_reader :options
+
+      # @return [Locale] The locale of the contents of this resource file
+      attr_accessor :locale
+
+      # @return [Pathname] The path to this resource file
+      attr_accessor :path
+
+      # @return [boolean] True if this resource format is enabled
+      def self.enabled?
+        true
+      end
 
-  # Get the value of a property
-  # @param key [String] The name of the property
-  # @return [String] The value of the property
-  def get(key)
-    @properties[key]
-  end
+      # @return [String] Preferred character to use to separate the locale
+      #   from the file basename
+      def self.preferred_locale_separator
+        '.'
+      end
 
-  # Set a property
-  # @param key [String] The name of the property
-  # @param value [String] The new value of the property
-  # @return [String] The new value of the property
-  def set(key, value)
-    @properties[key] = value
-  end
+      # @return [Array<String>] File extensions supported by this resource file
+      def self.extensions
+        raise 'extensions must be implemented by subclass'
+      end
 
-  # @return [String] String representation of this file
-  def to_s
-    path.basename.to_s
-  end
+      # @return [boolean] True if the file format consists of keys and values
+      def self.key_value?
+        false
+      end
 
-  def type
-    self.class.name.demodulize
-  end
+      # @return [boolean] true if this resource file supports
+      #   variable interpolation
+      def self.supports_variable_interpolation?
+        false
+      end
 
-  # @return [Array<String>] All property values split into sentences
-  def sentences
-    sentences = []
-    properties.values.each do |value|
-      string = Translatomatic::String.new(value, locale)
-      sentences += string.sentences
-    end
-    sentences
-  end
+      # Create a new resource file or load an existing file.
+      # If options[:locale] is unspecified, attempts to determine
+      # the locale of the file automatically, and if that fails,
+      # uses the default locale.
+      # Raises an exception if errors were encountered loading the file.
+      # @param path [String] Path to the file
+      # @param options [Hash<Symbol,String>] Options
+      # @return [Translatomatic::ResourceFile::Base] the resource file.
+      def initialize(path = nil, options = {})
+        raise 'expected options hash' if options && !options.is_a?(Hash)
+        raise t('file.unsupported', file: path) unless self.class.enabled?
+        initialize_attributes(path, options)
+        update_locale
+        init
+        try_load
+      end
 
-  # Create an interpolated variable string.
-  # @return [String] A string representing the interpolated variable, or
-  #   nil if this resource file doesn't support variable interpolation.
-  def create_variable(name)
-    return nil unless supports_variable_interpolation?
-    raise "create_variable(name) must be implemented by subclass"
-  end
+      # Save the resource file.
+      # @param target [Pathname] The destination path
+      # @param options [Hash<Symbol, Object>] Output format options
+      # @return [void]
+      def save(target = path, options = {})
+        raise 'save(path) must be implemented by subclass'
+      end
 
-  # @return [Regexp] A regexp used to match interpolated variables
-  def variable_regex
-    return nil unless supports_variable_interpolation?
-    raise "variable_regex must be implemented by subclass"
-  end
+      # @return [Symbol] The type of this resource file, e.g. ":properties"
+      def type
+        self.class.name.demodulize.downcase.to_sym
+      end
 
-  private
+      # Create a path for the current resource file with a given locale
+      # @param target_locale [String] The target locale
+      # @return [Pathname] The path of this resource file modified
+      #   for the given locale
+      def locale_path(target_locale)
+        modify_path_locale(path, target_locale,
+                           self.class.preferred_locale_separator)
+      end
 
-  include Translatomatic::Util
+      # Set all properties
+      # @param properties [Hash<String,String>] New properties
+      def properties=(properties)
+        # use set rather that set @properties directly as subclasses
+        # override set()
+        properties.each do |key, value|
+          set(key, value)
+        end
+      end
 
-  # called by constructor before load
-  def init
-  end
+      # @return [Hash<String,String>] key -> value properties
+      def properties
+        @properties.dup
+      end
 
-  # load contents from @path
-  def load
-    raise "load must be implemented by subclass"
-  end
+      # Get the value of a property
+      # @param key [String] The name of the property
+      # @return [String] The value of the property
+      def get(key)
+        @properties[key]
+      end
 
-  def update_locale
-    locale = @options[:locale] || detect_locale || Translatomatic::Locale.default
-    @locale = Translatomatic::Locale.parse(locale)
-    raise t("file.unknown_locale") unless @locale && @locale.language
-  end
+      # Get context of a property
+      # @param key [String] The name of the property
+      # @return [Array<String>] The property context, may be nil
+      def get_context(key)
+        @metadata.get_context(key)
+      end
 
-  def created_by
-    options = {
-      app: "Translatomatic",
-      version: Translatomatic::VERSION,
-      date: I18n.l(DateTime.now),
-    }
-    # use created by string in current file's locale, fall back to
-    # english locale if translation is missing.
-    t("file.created_by", options.merge({
-      locale: locale.language,
-      default: t("file.created_by", options.merge(locale: "en"))
-      })
-    )
-  end
+      # Set a property
+      # @param key [String] The name of the property
+      # @param value [String] The new value of the property
+      # @return [String] The new value of the property
+      def set(key, value)
+        @properties[key] = value
+      end
 
-  def read_contents(path)
-    File.read(path.to_s, mode: "r:bom|utf-8")
-  end
+      # @return [String] String representation of this file
+      def to_s
+        relative_path(path).to_s
+      end
 
-  def parsing_error(error)
-    raise Exception.new(error)
-  end
+      # @return [Array<Text>] All property values split into sentences
+      def sentences
+        sentences = []
+        properties.each_value do |value|
+          string = build_text(value, locale)
+          sentences += string.sentences
+        end
+        sentences
+      end
 
-  # detect locale from filename
-  def detect_locale
-    return nil unless path
-    tag = nil
-    basename = path.sub_ext('').basename.to_s
-    directory = path.dirname.basename.to_s
-    extlist = extension_list
+      # Create an interpolated variable string.
+      # @param name [String] The variable name
+      # @return [String] A string representing the interpolated variable, or
+      #   nil if this resource file doesn't support variable interpolation.
+      def create_variable(name)
+        return nil unless self.class.supports_variable_interpolation?
+        raise 'create_variable(name) must be implemented by subclass'
+      end
 
-    if basename.match(/_([-\w]{2,})$/i)
-      # locale after underscore in filename
-      tag = $1
-    elsif directory.match(/^([-\w]+)\.lproj$/)
-      # xcode localized strings
-      tag = $1
-    elsif extlist.length >= 2 && loc_idx = find_locale(extlist)
-      # multiple parts to extension, e.g. index.html.en
-      tag = extlist[loc_idx]
-    elsif valid_locale?(basename)
-      # try to match on entire basename
-      # (support for rails en.yml)
-      tag = basename
-    elsif valid_locale?(path.parent.basename)
-      # try to match on parent directory, e.g. strings/en-US/text.resw
-      tag = path.parent.basename
-    end
+      # @return [Regexp] A regexp used to match interpolated variables
+      def variable_regex
+        return nil unless self.class.supports_variable_interpolation?
+        raise 'variable_regex must be implemented by subclass'
+      end
 
-    tag ? Translatomatic::Locale.parse(tag, true) : nil
-  end
+      private
 
-  def valid_locale?(tag)
-    Translatomatic::Locale.new(tag).valid?
-  end
+      include Translatomatic::Util
+      include Translatomatic::Flattenation
+      include Translatomatic::PathUtils
+      include Translatomatic::DefineOptions
 
-  # test if the list of strings contains a valid locale
-  # return the index to the locale, or nil if no locales found
-  def find_locale(list)
-    list.find_index { |i| valid_locale?(i) }
-  end
+      # called by constructor before load
+      def init; end
 
-  # ext_sub() only removes the last extension
-  def strip_extensions
-    filename = path.basename.to_s
-    filename.sub!(/\..*$/, '')
-    path.parent + filename
-  end
+      # load contents from @path
+      def load
+        raise 'load must be implemented by subclass'
+      end
 
-  # for index.html.de, returns ['html', 'de']
-  def extension_list
-    filename = path.basename.to_s
-    idx = filename.index('.')
-    idx && idx < filename.length - 1 ? filename[idx + 1..-1].split('.') : []
-  end
+      def try_load
+        return unless @path
+        raise t('file.not_found', file: path) unless @path.exist?
+        load
+      end
 
-  # flatten hash or array of hashes to a hash of key => value pairs
-  def flatten(data)
-    result = {}
+      def initialize_attributes(path, options)
+        @options = options || {}
+        @properties = {}
+        @metadata = Metadata.new
+        @path = path.nil? || path.is_a?(Pathname) ? path : Pathname.new(path)
+      end
 
-    if data.kind_of?(Hash)
-      data.each do |key, value|
-        flatten_add(result, key, value)
+      def update_locale
+        default = Translatomatic::Locale.default
+        locale = @options[:locale] || detect_path_locale(path) || default
+        @locale = Translatomatic::Locale.parse(locale)
+        raise t('file.unknown_locale') unless @locale && @locale.language
       end
-    elsif data.kind_of?(Array)
-      data.each_with_index do |value, i|
-        key = "key" + i.to_s
-        flatten_add(result, key, value)
+
+      def created_by
+        options = {
+          app: 'Translatomatic',
+          version: Translatomatic::VERSION,
+          date: I18n.l(Time.now),
+          locale: locale.language,
+          url: Translatomatic::URL
+        }
+        t('file.created_by', options)
       end
-    end
 
-    result
-  end
+      def created_by?
+        @created_by
+      end
 
-  def flatten_add(result, key, value)
-    if needs_flatten?(value)
-      children = flatten(value)
-      children.each do |ck, cv|
-        result[key + "." + ck] = cv
+      def parsing_error(error)
+        raise StandardError, error
       end
-    else
-      result[key] = value
     end
   end
-
-  def needs_flatten?(value)
-    value.kind_of?(Array) || value.kind_of?(Hash)
-  end
 end