air18n 0.0.1

data/lib/air18n/less_silly_chain.rb

@@ -0,0 +1,54 @@
+require 'i18n'
+
+module Air18n
+  # I18n::Backend::Chain has the rather silly behavior that options[:default] is
+  # removed from the options hash for all Backends except for the last. It's
+  # not clear why, know why because this behavior is not documented.
+  # LessSillyChain works just like Backend::Chain but without this behavior.
+  class LessSillyChain < I18n::Backend::Chain
+    def translate(locale, key, options = {})
+      namespace = nil
+
+      rescued_exception = nil
+      caught_exception = nil
+      backends.each do |backend|
+        rescued_exception = nil
+        caught_exception = nil
+
+        # We jump through two hoops to make this work with both i18n-0.5.0 and
+        # i18n-0.6.0.
+        #   - i18n-0.5.0: Backends raise TranslationMissingData exception when
+        #     a translation is missing
+        #   - i18n-0.6.0: Backends throw TranslationMissing "exceptions" to the
+        #     :exception tag when a translation is missing.
+        #
+        # To make both happy, we both rescue and catch.
+        caught_exception = catch(:exception) do
+          begin
+            translation = backend.translate(locale, key, options.merge(:raise => true))
+            if namespace_lookup?(translation, options)
+              namespace ||= {}
+              namespace.merge!(translation)
+            elsif !translation.nil?
+              return translation
+            end
+          rescue Exception => e
+            rescued_exception = e
+          end
+
+          # Nothing caught.
+          nil
+        end
+      end
+
+      return namespace if namespace
+
+      if caught_exception
+        throw(:exception, caught_exception)
+      end
+      if rescued_exception
+        raise(rescued_exception)
+      end
+    end
+  end
+end

data/lib/air18n/logging_helper.rb

@@ -0,0 +1,13 @@
+module Air18n
+  module LoggingHelper
+    module_function
+
+    def info text
+      ActiveRecord::Base.logger.info text
+    end
+
+    def error text
+      ActiveRecord::Base.logger.error text
+    end
+  end
+end

data/lib/air18n/mock_priority.rb

@@ -0,0 +1,25 @@
+module Air18n
+  class MockPriority
+    include Priority
+    def initialize(key_usage = {})
+      @key_usage = key_usage
+    end
+
+    def key_used(key)
+      @key_usage[key] ||= 0
+      @key_usage[key] += 1
+    end
+
+    def key_usage
+      @key_usage
+    end
+
+    def self.mock_priority(key_usage)
+      @old_priority = I18n.priority
+      mock_priority = Air18n::MockPriority.new(key_usage)
+      I18n.priority = mock_priority
+      yield mock_priority
+      I18n.priority = @old_priority
+    end
+  end
+end

data/lib/air18n/phrase.rb

@@ -0,0 +1,92 @@
+module Air18n
+  class Phrase < ActiveRecord::Base
+    has_many :phrase_translations, :dependent=>:delete_all
+    has_many :phrase_screenshots, :foreign_key => 'phrase_key', :primary_key => 'key', :dependent=>:delete_all
+
+    after_update :mark_translations_stale
+
+    validates_uniqueness_of :key
+
+    def mark_translations_stale
+      translations = PhraseTranslation.find_all_by_phrase_id(id)
+      translations.each do |translation|
+        translation.is_stale = true
+
+        # Skip validations. This is because for historical reasons some
+        # translations are invalid. We want to mark these stale regardless.
+        translation.save!(:validate => false)
+      end
+    end
+
+    def self.by_key lookup
+      if !@by_key then
+        @by_key = {}
+        Phrase.all.group_by{|e| e.key}.each_pair do |k,v|
+          if v.is_a? Array
+            @by_key[k] = v.first
+          else
+          end
+        end
+      end
+
+      val = @by_key[lookup]
+      val
+    end
+
+    # phrases that have no corresponding phrase translations by human
+    scope :needing_first_translation, lambda{|locale| where("phrases.id NOT IN (select phrase_id from phrase_translations WHERE locale='#{locale}' AND is_latest = ?)", true) }
+
+    scope :needing_updated_translation, lambda{|locale|
+      joins(:phrase_translations).where("phrase_translations.locale = '#{locale}' AND phrase_translations.is_latest = ? AND phrase_translations.is_stale = ?", true, true)
+    }
+
+    scope :not_needing_updated_translation, lambda{|locale|
+      joins(:phrase_translations).where("phrase_translations.locale = '#{locale}' AND phrase_translations.is_latest = ? AND phrase_translations.is_stale = ?", true, false)
+    }
+
+    scope :needing_first_or_updated_translation, lambda{|locale|
+      pull_in_phrase_translations.where(
+        "phrases.id NOT IN (select phrase_id from phrase_translations WHERE locale='#{locale}' AND is_latest = ?) OR (phrase_translations.is_latest = ? AND phrase_translations.is_stale = ? AND phrase_translations.locale = '#{locale}')", true, true, true
+      )
+    }
+
+    scope :pull_in_phrase_translations, lambda{|locale|
+      joins(
+        "LEFT OUTER JOIN phrase_translations ON phrase_translations.phrase_id = phrases.id AND phrase_translations.locale = '#{locale}' AND phrase_translations.is_latest = #{connection.quoted_true}"
+      )
+    }
+
+    scope :pull_in_all_phrase_translations, lambda { joins(
+      "LEFT OUTER JOIN phrase_translations ON phrase_translations.phrase_id = phrases.id AND phrase_translations.is_latest = #{connection.quoted_true}"
+    ) }
+
+    scope :pull_in_all_up_to_date_phrase_translations, lambda { joins(
+      "LEFT OUTER JOIN phrase_translations ON phrase_translations.phrase_id = phrases.id AND phrase_translations.is_latest = #{connection.quoted_true} AND phrase_translations.is_stale = #{connection.quoted_false}"
+    ) }
+
+    # The still_used scope is wrapped in a lambda expression to
+    # I18n.still_used_phrase_ids is reevaluated each time.
+    # This makes dynamic updating possible and testing easier.
+    scope :still_used, lambda { where('phrases.id IN (?)', I18n.still_used_phrase_ids) }
+    scope :no_longer_used, lambda { where('phrases.id NOT IN (?)', I18n.still_used_phrase_ids) }
+
+    scope :rich_text, lambda { where("phrases.value LIKE '%<p>%'") }
+
+    # Fetches text of the most recent translation, if available.
+    def latest_translation locale
+      PhraseTranslation.where(:phrase_id => id, :locale => locale, :is_latest => true).first
+    end
+
+    def variables
+      PhraseTranslation.detect_variables(value)
+    end
+
+    def self.is_rich?(text)
+      text.include?('<p>')
+    end
+
+    def is_rich_text?
+      !!(self.value && Phrase.is_rich?(self.value))
+    end
+  end
+end

data/lib/air18n/phrase_screenshot.rb

@@ -0,0 +1,76 @@
+module Air18n
+  class PhraseScreenshot < ActiveRecord::Base
+    validates_presence_of :phrase_key
+    validates_uniqueness_of :phrase_key, :scope => [:controller, :action]
+    belongs_to :phrase, :foreign_key => 'phrase_key', :primary_key => 'key'
+    scope :positioned, :conditions => "width IS NOT NULL AND height IS NOT NULL and x IS NOT NULL and y IS NOT NULL and width != 0 and height != 0 and x != 0 and y != 0"
+    scope :unpositioned, :conditions => "width IS NULL OR height IS NULL OR x IS NULL OR y IS NULL OR width = 0 OR height = 0 OR x = 0 OR y = 0"
+    belongs_to :screenshot, :foreign_key => 'screenshot_url', :primary_key => 'screenshot_url'
+
+    after_create :create_row_in_screenshots
+
+    # Returns 2D array of phrase_keys => array of routes_contexts for which we have screenshots
+    def self.all_phrase_urls
+      Hash.new { |h, k| h[k] = [] }.tap do |all_phrase_urls|
+        # Uses raw SQL for speed.
+        connection.select_all("SELECT `phrase_key`, `action`, `controller` FROM phrase_screenshots").each do |record|
+          all_phrase_urls[record['phrase_key']] << make_routes_context(record['controller'], record['action'])
+        end
+      end
+    end
+
+    def has_position?
+      return width.present? && height.present? && x.present? && y.present? &&
+        width > 0 && height > 0 && x > 0 && y > 0
+    end
+
+    def routes_context
+      self.class.make_routes_context(self.controller, self.action)
+    end
+
+    def self.make_routes_context(controller, action)
+      "#{controller}-#{action}"
+    end
+
+    def create_row_in_screenshots
+      if !Screenshot.find_by_screenshot_url(self.screenshot_url)
+        screenshot = Screenshot.create(:screenshot_url => self.screenshot_url, :status => 0)
+      end
+    end
+
+    def self.get_screenshots_ordered_by_relevance_for_many_phrases(phrases, controller_name, action_name)
+      screenshots = PhraseScreenshot.includes(:screenshot).where(:phrase_key => phrases.collect(&:key))
+      screenshots_for_each_phrase = Hash.new { |h, k| h[k] = [] }
+      screenshots.all.each { |screenshot| screenshots_for_each_phrase[screenshot.phrase_key] << screenshot }
+      screenshots_for_each_phrase.inject(Hash.new([])) do |carry, (key, screenshots_for_phrase)|
+        carry[key] = self.get_screenshots_ordered_by_relevance(screenshots_for_phrase, controller_name, action_name)
+      carry
+      end
+    end
+
+    # Returns the screenshots associated with phrase, with the screenshots for the
+    # specified controller and action first in the list.
+    # If controller_name or action_name are nil, result is unordered.
+    def self.get_screenshots_ordered_by_relevance(screenshots_for_a_phrase, controller_name, action_name)
+      if controller_name || action_name
+        screenshots_for_a_phrase.sort_by do |x|
+          priority = 0
+          priority -= 1 if x.action == action_name
+          priority -= 2 if x.controller == controller_name
+          priority
+        end
+      else
+        screenshots_for_a_phrase
+      end
+    end
+
+    def self.serialize_page_id(controller, action)
+      "#{controller}##{action}"
+    end
+
+    def self.deserialize_page_id(page_id)
+      controller, action = page_id.split("#")
+      Struct.new(:controller, :action).new(controller, action)
+    end
+  end
+end

data/lib/air18n/phrase_translation.rb

@@ -0,0 +1,348 @@
+require 'air18n/logging_helper'
+
+module Air18n
+  class PhraseTranslation < ActiveRecord::Base
+    belongs_to :phrase
+    belongs_to :user
+
+    validates_presence_of :key, :locale
+
+    scope :locale, lambda {|loc| { :conditions => ['locale = ?', loc.to_s] } }
+    scope :latest, lambda { where(:is_latest => true) }
+    scope :up_to_date, lambda { where(:is_stale => false) }
+
+    before_create :set_latest
+
+    validate :check_matching_variables
+
+    # Sets is_latest of this translation, and removes the is_latest flag from all
+    # previous translations.
+    def set_latest
+      self.is_latest = true
+      other_translations = PhraseTranslation.find_all_by_phrase_id_and_locale(phrase_id, locale)
+      other_translations.each do |other|
+        if other.id != id && other.is_latest
+          other.is_latest = false
+
+          # Skip validations. This is because in the case where a translator is
+          # making a new (valid) translation where the previous translation was
+          # invalid, we want to be able to still mark the old translation as
+          # invalid.
+          other.save!(:validate => false)
+        end
+      end
+    end
+
+    COST_PER_WORD_TRANSLATION = 0.07
+    COST_PER_WORD_VERIFICATION = 0.05
+
+    # Provides a complete set of latest translations for specified locales, in nested hash format.
+    # filter_opts are passed to keep_key? for optional filtering, like throwing
+    # away user-generated content.
+    def self.translations_for_locales(locales, filter_opts={})
+
+      # set up the hashes we want
+      all_locales = {}
+      locales.each do |loc|
+        data = translations_for_locale(loc, filter_opts)
+        all_locales[loc] = data
+      end
+      all_locales
+    end
+
+    def self.translator_activity_data user_id=0, opts={}
+      user_criterion = user_id > 0 ? "AND pt.user_id=#{user_id}" : "AND NOT pt.user_id=0"
+      since_criterion = "AND pt.created_at >= '#{opts[:since].to_formatted_s(:db)}}'" if opts[:since]
+      sql = "SELECT year(pt.created_at) year, month(pt.created_at) month, pt.user_id user_id, pt.locale locale, day(pt.created_at) day, p.id phrase_id, p.value phrase_value, pt.is_verification FROM phrase_translations pt, phrases p WHERE pt.phrase_id=p.id #{user_criterion} #{since_criterion} GROUP BY year(pt.created_at), month(pt.created_at), pt.user_id, pt.locale, pt.phrase_id"
+      res = self.connection.select_rows(sql)
+      phrases_per_user_locale_month_year =
+        Hash.new {|h, year| h[year] =
+          Hash.new {|h, month| h[month] =
+            Hash.new{|h, user_id| h[user_id] =
+              Hash.new{|h, locale| h[locale] =
+                Hash.new {|h, day| h[day] =
+                  Hash.new {|h, is_verification| h[is_verification] =
+                    Hash.new
+      } } } } } }
+
+      res.each do |row|
+        year, month, user_id, locale, day, phrase_id, phrase_value, is_verification = row
+        is_verification = (is_verification == 1)
+        phrases_per_user_locale_month_year[year][month][user_id][locale][day][is_verification].merge!({phrase_id => phrase_value})
+      end
+
+      activities = []
+      phrases_per_user_locale_month_year.each do |year, months|
+        months.each do |month, user_ids|
+          user_ids.each do |user_id, locales|
+            locales.each do |locale, days|
+              if opts[:daily]
+                days.each do |day, phrase_ids_values_by_type|
+                  activities << {:year => year, :month => month, :day => day, :locale => locale, :user_id => user_id, :activity => construct_activity(phrase_ids_values_by_type)}
+                end
+              else
+                monthly_phrase_ids_values_by_type = { false => {}, true => {} }
+                days.each do |_, phrase_ids_values_by_type|
+                  [false, true].each do |is_verification|
+                    monthly_phrase_ids_values_by_type[is_verification].merge!(phrase_ids_values_by_type[is_verification])
+                  end
+                end
+                activity = construct_activity(monthly_phrase_ids_values_by_type)
+                translation_ids = monthly_phrase_ids_values_by_type[false].keys
+                if translation_ids.empty?
+                  activity[:num_phrases_prev_translated] = 0
+                else
+                  activity[:num_phrases_prev_translated] = self.count_by_sql("select count(distinct(phrase_id)) from phrase_translations where user_id = #{user_id} and phrase_id in (#{translation_ids.join(',')}) and locale = '#{locale}' and created_at < '#{year}-#{month}-01'")
+                end
+                activities << {:year => year, :month => month, :locale => locale, :user_id => user_id, :activity => activity}
+              end
+            end
+          end
+        end
+      end
+      activities.sort do |a,b|
+        if a[:year] == b[:year]
+          if a[:month] == b[:month]
+            if a[:user_id] == b[:user_id]
+              if a[:locale] == b[:locale]
+                if !a[:day] || !b[:day] || a[:day] == b[:day]
+                  0
+                else
+                  a[:day] <=> b[:day]
+                end
+              else
+                a[:locale] <=> b[:locale]
+              end
+            else
+              a[:user_id] <=> b[:user_id]
+            end
+          else
+            a[:month] <=> b[:month]
+          end
+        else
+          a[:year] <=> b[:year]
+        end
+      end
+    end
+
+    # Helper method for translator_activity_data which counts words and payment
+    # of a set of translations and verifications, in (is verification bool) => (phrase id =>
+    # English phrase value map).
+    def self.construct_activity(phrase_ids_values_by_type)
+      {}.tap do |activity|
+        activity[:num_translations], activity[:word_count_translations] = translation_word_count(phrase_ids_values_by_type[false])
+        activity[:num_verifications], activity[:word_count_verifications] = translation_word_count(phrase_ids_values_by_type[true])
+        activity[:payment] = activity[:word_count_translations] * COST_PER_WORD_TRANSLATION + activity[:word_count_verifications] * COST_PER_WORD_VERIFICATION
+      end
+    end
+
+    # Helper method for construct_activity which counts words in a set of
+    # translations or verifications, in phrase id => English phrase value map.
+    def self.translation_word_count(phrase_ids_to_values)
+      [phrase_ids_to_values.count, phrase_ids_to_values.values.join(' ').scan(/\w+/).size]
+    end
+
+    def self.activity_for_user_id uid, opts
+      translator_activity_data uid, opts
+    end
+
+    # Returns an array of integers representing user IDs who are not admins but
+    # who we also do not want to pay by the word for translations.
+    def self.fetch_unpaid_nonadmin_translator_list
+      rich_memory = RichMemory.find_by_memory_type(RichMemory::TYPE_TRANSLATOR_LIST)
+      if rich_memory
+        rich_memory.payload
+      else
+        []
+      end
+    end
+
+    # Adds or removes a user from the unpaid-non-admin translator list.
+    def self.set_nonadmin_translator_payment_status id, should_be_paid
+      set = fetch_unpaid_nonadmin_translator_list.to_set
+      if should_be_paid
+        set -= [id]
+      else
+        set += [id]
+      end
+      write_unpaid_nonadmin_translator_list(set.to_a)
+    end
+
+    # translator_list: array of integers representing user IDs.
+    # Throws an exception if saving fails.
+    def self.write_unpaid_nonadmin_translator_list translator_list
+      rich_memory = RichMemory.find_by_memory_type(RichMemory::TYPE_TRANSLATOR_LIST) || RichMemory.new(:memory_type => RichMemory::TYPE_TRANSLATOR_LIST)
+      rich_memory.payload = translator_list
+      rich_memory.save!
+    end
+
+    def self.translator_payment_data
+      today_last_month = 1.month.ago
+      ret = []
+      PhraseTranslation.translator_activity_data(0, :since => 2.months.ago).each do |stats|
+        if stats[:year] == today_last_month.year && stats[:month] == today_last_month.month && stats[:user_id] > 0
+          user = User.find_by_id(stats[:user_id])
+          stats[:smart_name] = user.try(:smart_name)
+
+          stats[:currency] = 'EUR'
+          stats[:is_admin] = user.try(:role_admin?)
+          stats[:is_paid] = !stats[:is_admin] && !fetch_unpaid_nonadmin_translator_list.include?(stats[:user_id])
+          stats[:usd_amount] = Currency.convert(stats[:activity][:payment], stats[:currency], "USD").round
+          if stats[:is_paid] && stats[:usd_amount] > 0
+            stats[:description] = "Translator payout "
+            stats[:description] << "for #{Date::MONTHNAMES[today_last_month.month]} #{today_last_month.year}. "
+            stats[:description] << "(#{stats[:activity][:word_count_translations]} translated words and #{stats[:activity][:word_count_verifications]} verified words in #{stats[:activity][:num_translations]} translated and #{stats[:activity][:num_verifications]} verified #{Airbnb::I18n.decode_locale(stats[:locale])} phrases.)"
+
+            stats[:line_item] = LineItem.new(
+              :user_id => stats[:user_id],
+              :ready_for_release_at => Date.today,
+              :sub_type => LineItem::SUB_TYPE_TRANSLATION_PAYOUT,
+              :native_currency => stats[:currency],
+              :amount => stats[:usd_amount],
+              :item => nil,
+              :description => (stats[:description] + " Thank you!!")
+            )
+          end
+          ret << stats
+        end
+      end
+      ret
+    end
+
+    # Returns all translations for a locale. filter_opts are passed on to
+    # self.keep_key? for filtering.
+    def self.translations_for_locale(loc, filter_opts={})
+      data = {}
+      case loc
+      when :en
+        # Uses a raw unbatched SQL query for speed.
+        Phrase.connection.select_all("SELECT `key`, `value` FROM phrases").each do |record|
+          data[record['key']] = record['value'] if keep_key?(record['key'], filter_opts)
+        end
+      else
+        # Uses a raw unbatched SQL query for speed.
+        PhraseTranslation.connection.select_all(PhraseTranslation.select("`key`, `value`").latest.where(:locale => loc).to_sql).each do |record|
+          data[record['key']] = record['value'] if keep_key?(record['key'], filter_opts)
+        end
+      end
+      data
+    end
+
+    def self.keep_key?(key, filter_opts={})
+      if filter_opts[:exclude_ugc] && I18n.phrase_key_is_ugc?(key)
+        return false
+      end
+      if filter_opts[:exclude_unused] && !I18n.still_used_keys_hash.include?(key)
+        return false
+      end
+      if filter_opts[:exclude_all]
+        return false
+      end
+
+      true
+    end
+
+    def check_matching_variables
+      unless variables_match?
+        our_variables = self.variables
+        their_variables = self.phrase.variables
+        our_extra = our_variables - their_variables
+        their_extra = their_variables - our_variables
+        problems = []
+        def quote(vars)
+          vars.map { |var| "%{#{var}}" }.join(", ")
+        end
+        unless their_extra.empty?
+          problems << "Var #{quote their_extra} missing from translation"
+        end
+        unless our_extra.empty?
+          problems << "Var #{quote our_extra} should not be in translation"
+        end
+        unless problems.empty?
+          self.errors.add(:value, problems.join('; '))
+        end
+      end
+    end
+
+    def variables_match?
+      self.variables == self.phrase.variables
+    end
+
+    def variables
+      PhraseTranslation.detect_variables(value)
+    end
+
+    def self.detect_variables(search_in)
+      case search_in
+      when String then Set.new(search_in.scan(/\{\{(\w+)\}\}/).flatten + search_in.scan(/\%\{(\w+)\}/).flatten)
+      when Array then search_in.inject(Set[]) { |carry, item| carry + detect_variables(item) }
+      when Hash then search_in.values.inject(Set[]) { |carry, item| carry + detect_variables(item) }
+      else Set[]
+      end
+    end
+
+    # For every translated key in a locale, returns list of ids
+    # of PhraseTranslations that are the most recent translation of a key.
+    def self.list_of_latest_translations(locale)
+      latest_translations = PhraseTranslation.select("max(id) as max_id").where("locale='#{locale}'").group(:phrase_id).collect{|e| e.max_id}
+      latest_translations.empty? ? [0] : latest_translations
+    end
+
+    def verification?
+      is_verification
+    end
+
+    def latest?
+      is_latest
+    end
+
+    def stale?
+      is_stale
+    end
+
+    # For every PhraseTranslation in Airbnb::I18n::TRANSLATED_LOCALES, resets
+    # is_latest and is_stale columns based on phrases.updated_at and
+    # phrase_translations.created_at dates.
+    #
+    # This is used by the migration that adds these columns.
+    def self.reset_latest_and_stale_flags_from_timestamps
+      phrase_id_to_updated_at = {}
+      Phrase.find_each do |phrase|
+        phrase_id_to_updated_at[phrase.id] = phrase.updated_at
+      end
+
+      Airbnb::I18n::TRANSLATED_LOCALES.each do |locale|
+        # Output progress to console in case of running this on console.
+        puts "Resetting flags for #{locale}..."
+        latest_translations = PhraseTranslation.select("max(id) as max_id").where("locale='#{locale}'").group(:phrase_id).collect{|e| e.max_id}.to_set
+
+        PhraseTranslation.where(:locale => locale).find_each do |translation|
+          phrase_updated_at = phrase_id_to_updated_at[translation.phrase_id]
+          if !phrase_updated_at
+            # This happens for a few phrases, very weirdly. We could destroy them
+            # at some point.
+            LoggingHelper.error "What the?! Phrase translation #{translation.inspect} has no corresponding phrase."
+            next
+          end
+          if phrase_updated_at > translation.created_at
+            translation.is_stale = true
+          end
+          if latest_translations.include?(translation.id)
+            translation.is_latest = true
+          end
+          translation.save!(:validate => false)
+        end
+      end
+    end
+
+    def verification_allowed?(latest_translation)
+      if latest_translation.is_verification? && !latest_translation.stale?
+        return [false, "Translation already verified; nothing saved."]
+      end
+      if user_id == latest_translation.user_id && !latest_translation.stale?
+        return [false, "You last translated this phrase, so somebody else must verify it."]
+      end
+      return [true]
+    end
+  end
+end