phraseapp_updater 0.1.7 → 2.0.0

data/bin/synchronize_phraseapp.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+. "${SCRIPTPATH}/phraseapp_common.sh"
+
+if [ ! -d ".git" ]; then
+    echo "Error: must be run in a git checkout root" >&2
+    exit 1
+fi
+
+# Configuration is via environment, and expected to be provided from a Ruby
+# driver.
+for variable in PHRASEAPP_API_KEY PHRASEAPP_PROJECT_ID BRANCH REMOTE PREFIX FILE_FORMAT NO_COMMIT; do
+    if [ -z "${!variable}" ]; then
+        echo "Error: must specify $variable" >&2
+        exit 1
+    fi
+done
+
+# Ensure we're up to date
+git fetch "${REMOTE}"
+
+current_branch=$(git rev-parse "${REMOTE}/${BRANCH}")
+
+# First, fetch the current contents of PhraseApp's staged ("verified") state.
+current_phraseapp_path=$(make_temporary_directory)
+common_ancestor=$(phraseapp_updater download "${current_phraseapp_path}" \
+                      --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
+                      --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
+                      --file_format="${FILE_FORMAT}")
+
+# If common_ancestor is not available or reachable from BRANCH, we've been
+# really naughty and rebased without uploading the result to PhraseApp
+# afterwards. If it's not available, we lose: the best we can do is manually
+# perform a 2-way diff and force upload to phraseapp. If it's still available
+# but not reachable, we can still try and perform a 3 way merge, but the results
+# will not be as accurate: warn the user.
+if ! git cat-file -e "${common_ancestor}^{commit}"; then
+    echo "Common ancestor commit could not be found: was '${BRANCH}' rebased without updating PhraseApp?" >&2
+    exit 1
+elif ! git merge-base --is-ancestor "${common_ancestor}" "${current_branch}"; then
+    echo "Warning: ancestor commit was not reachable from '${BRANCH}': 3-way merge may be inaccurate" >&2
+fi
+
+current_branch_path=$(extract_commit "${current_branch}")
+common_ancestor_path=$(extract_commit "${common_ancestor}")
+
+current_phraseapp_tree=$(make_tree_from_directory "${current_phraseapp_path}")
+
+# We have four cases to handle:
+# 1: PhraseApp and BRANCH locales both changed since common_ancestor:
+#  * Record current PhraseApp in a commit A with parent `common_ancestor`
+#  * 3-way merge BRANCH locales and PhraseApp contents
+#  * Commit the result to BRANCH with parents BRANCH and A, yielding B
+#  * Push the result to phraseapp with new common_ancestor B
+# 2: Only BRANCH changed since common_ancestor:
+#  * Push BRANCH locales to PhraseApp with new common_ancestor BRANCH
+# 3: Only PhraseApp changed since common_ancestor:
+#  * Commit and push PhraseApp contents to BRANCH yielding commit A
+#  * Update PhraseApp common_ancestor to A
+# 4: Neither changed:
+#  * Do nothing
+
+phraseapp_changed=$(if locales_changed "${common_ancestor_path}" "${current_phraseapp_path}"; then echo t; else echo f; fi)
+branch_changed=$(if locales_changed "${common_ancestor_path}" "${current_branch_path}"; then echo t; else echo f; fi)
+
+if [ "${phraseapp_changed}" = 't' ] && [ "${branch_changed}" = 't' ]; then
+    echo "$BRANCH branch and PhraseApp both changed: 3-way merging" >&2
+
+    # 3-way merge
+    merge_resolution_path=$(make_temporary_directory)
+    phraseapp_updater merge "${common_ancestor_path}" "${current_branch_path}" "${current_phraseapp_path}" \
+                      --to "${merge_resolution_path}" \
+                      --file_format="${FILE_FORMAT}"
+
+    if [ "$NO_COMMIT" != 't' ]; then
+        # Create a commit to record the pre-merge state of PhraseApp
+        phraseapp_commit=$(git commit-tree "${current_phraseapp_tree}" \
+                               -p "${common_ancestor}" \
+                               -m "Remote locale changes made on PhraseApp (drop when rebasing)")
+
+        # Commit merge result to PREFIX in BRANCH
+        merge_resolution_tree=$(make_tree_from_directory "${merge_resolution_path}")
+        merged_branch_tree=$(replace_nested_tree "${current_branch}^{tree}" "${PREFIX}" "${merge_resolution_tree}")
+
+        merge_commit=$(git commit-tree "${merged_branch_tree}" \
+                           -p "${current_branch}" \
+                           -p "${phraseapp_commit}" \
+                           -m "Merged locale changes from PhraseApp")
+
+        # Push to BRANCH
+        git push "${REMOTE}" "${merge_commit}:refs/heads/${BRANCH}"
+        new_parent_commit="${merge_commit}"
+    else
+        # Merge is only to phraseapp: record current branch as new common ancestor
+        echo "Not committing to $BRANCH" >&2
+        new_parent_commit="${current_branch}"
+    fi
+
+    # Push merge result to phraseapp
+    phraseapp_updater upload "${merge_resolution_path}" \
+                      --parent_commit="${new_parent_commit}" \
+                      --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
+                      --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
+                      --file_format="${FILE_FORMAT}"
+
+elif [ "${branch_changed}" = 't' ]; then
+    echo "Only $BRANCH branch changed: updating PhraseApp" >&2
+
+    # Upload to phraseapp
+    phraseapp_updater upload "${current_branch_path}" \
+                      --parent_commit="${current_branch}" \
+                      --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
+                      --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
+                      --file_format="${FILE_FORMAT}"
+
+elif [ "${phraseapp_changed}" = 't' ]; then
+    if [ "$NO_COMMIT" != 't' ]; then
+        echo "Only PhraseApp changed: updating $BRANCH branch" >&2
+
+        updated_branch_tree=$(replace_nested_tree "${current_branch}^{tree}" "${PREFIX}" "${current_phraseapp_tree}")
+        update_commit=$(git commit-tree "${updated_branch_tree}" \
+                            -p "${current_branch}" \
+                            -m "Incorporate locale changes from PhraseApp")
+
+        git push "${REMOTE}" "${update_commit}:refs/heads/${BRANCH}"
+
+        # Set ancestor on phraseapp
+        phraseapp_updater update_parent_commit \
+                          --parent_commit="${update_commit}" \
+                          --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
+                          --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}"
+    else
+        echo "Only PhraseApp changed: not committing to $BRANCH branch" >&2
+    fi
+else
+    echo "No changes made since common ancestor" >&2
+fi

data/lib/phraseapp_updater.rb

@@ -1,176 +1,165 @@
+# frozen_string_literal: true
+
 require 'phraseapp_updater/version'
 require 'phraseapp_updater/index_by'
 require 'phraseapp_updater/differ'
 require 'phraseapp_updater/locale_file'
-require 'phraseapp_updater/locale_file/loader'
 require 'phraseapp_updater/phraseapp_api'
 require 'phraseapp_updater/yml_config_loader'
 
 class PhraseAppUpdater
   using IndexBy
 
-  def initialize(phraseapp_api_key, phraseapp_project_id, file_format)
+  def self.for_new_project(phraseapp_api_key, phraseapp_project_name, file_format, parent_commit)
+    api = PhraseAppAPI.new(phraseapp_api_key, nil, LocaleFile.class_for_file_format(file_format))
+    project_id = api.create_project(phraseapp_project_name, parent_commit)
+    return self.new(phraseapp_api_key, project_id, file_format), project_id
+  end
+
+  def initialize(phraseapp_api_key, phraseapp_project_id, file_format, default_locale: 'en')
     @locale_file_class = LocaleFile.class_for_file_format(file_format)
+    @default_locale    = default_locale
     @phraseapp_api     = PhraseAppAPI.new(phraseapp_api_key, phraseapp_project_id, @locale_file_class)
   end
 
-  def self.load_config(config_file_path)
-    YMLConfigLoader.new(config_file_path)
+  def diff_directories(our_path, their_path)
+    (our_locales, their_locales) = load_locale_directories(our_path, their_path)
+    diff_locale_files(our_locales, their_locales)
   end
 
-  def push(previous_locales_path, new_locales_path)
-    phraseapp_locales = @phraseapp_api.download_locales
+  def merge_directories(our_path, their_path, ancestor_path, result_path)
+    (our_locales, their_locales, ancestor_locales) =
+      load_locale_directories(our_path, their_path, ancestor_path)
 
-    phraseapp_files = load_phraseapp_files(phraseapp_locales, false)
+    merged_locales = merge_locale_files(our_locales, their_locales, ancestor_locales)
 
-    (previous_locale_files, new_locale_files) =
-      load_locale_files(previous_locales_path, new_locales_path)
+    write_locale_directory(result_path, merged_locales)
+  end
 
-    validate_files!([phraseapp_files, previous_locale_files, new_locale_files])
+  def merge_files(ours, theirs, ancestor, result)
+    our_file, their_file = load_locale_files(ours, theirs)
+    # Read the ancestor if provided
+    ancestor_file = load_locale_file(ancestor) unless ancestor.nil?
 
-    new_locale_files = new_locale_files.index_by(&:name)
-    phraseapp_files  = phraseapp_files.index_by(&:name)
+    result_file = merge_locale_files(our_file, their_file, ancestor_file)
 
-    resolved_files = previous_locale_files.map do |previous_locale_file|
-      new_locale_file = new_locale_files.fetch(previous_locale_file.name)
-      phraseapp_file  = phraseapp_files.fetch(previous_locale_file.name)
+    write_locale_file(result, result_file)
+  end
 
-      resolved_content = Differ.resolve!(original: previous_locale_file.parsed_content,
-                                         primary: new_locale_file.parsed_content,
-                                         secondary: phraseapp_file.parsed_content)
+  def upload_directory(path)
+    locales = load_locale_directory(path)
+    upload_locale_files(locales)
+  end
 
-      @locale_file_class.from_hash(previous_locale_file.name, resolved_content)
-    end
+  def download_to_directory(path)
+    locale_files = download_locale_files
+    write_locale_directory(path, locale_files)
+  end
 
-    # Upload all of the secondary languages first,
-    # so that the missing keys in them get filled in
-    # with blanks on PhraseApp by the default locale.
-    # If we do the clean up after uploading the default
-    # locale file, these blanks will get cleaned because
-    # they're not mentioned in the secondary locale files.
+  def update_parent_commit(parent_commit)
+    @phraseapp_api.update_parent_commit(parent_commit)
+  end
 
-    default_locale_file = find_default_locale_file(phraseapp_locales, resolved_files)
+  def read_parent_commit
+    @phraseapp_api.read_parent_commit
+  end
 
-    resolved_files.delete(default_locale_file)
+  private
 
-    changed_files = resolved_files.select do |file|
-      file.parsed_content != phraseapp_files[file.name].parsed_content
+  def diff_locale_files(our_locales, their_locales)
+    (our_content, their_content) = [our_locales, their_locales].map do |locales|
+      locales.each_with_object({}) do |locale, h|
+        h[locale.locale_name] = locale.parsed_content
+      end
     end
 
-    upload_ids = @phraseapp_api.upload_files(changed_files)
-    @phraseapp_api.remove_keys_not_in_uploads(upload_ids)
+    HashDiff.diff(our_content, their_content)
+  end
 
-    puts "Uploading #{default_locale_file}"
-    upload_id = @phraseapp_api.upload_file(default_locale_file)
+  def merge_locale_files(our_locales, their_locales, ancestor_locales)
+    ours      = our_locales.index_by(&:locale_name)
+    theirs    = their_locales.index_by(&:locale_name)
+    ancestors = ancestor_locales.index_by(&:locale_name)
 
-    puts "Removing keys not in upload #{upload_id}"
-    @phraseapp_api.remove_keys_not_in_upload(upload_id)
+    locale_names = Set.new.merge(ours.keys).merge(theirs.keys)
 
-    LocaleFileUpdates.new(phraseapp_files.values, [default_locale_file] + resolved_files)
+    locale_names.map do |locale_name|
+      our_file      = ours.fetch(locale_name)
+      their_file    = theirs.fetch(locale_name)
+      ancestor_file = ancestors[locale_name]
+      merge_locale_file(our_file, their_file, ancestor_file)
+    end
   end
 
-  def pull(fallback_locales_path)
-    phraseapp_locales = @phraseapp_api.download_locales
-
-    phraseapp_files_without_unverified = load_phraseapp_files(phraseapp_locales, true)
-    phraseapp_files_with_unverified    = load_phraseapp_files(phraseapp_locales, false)
-    fallback_files                     = load_locale_files(fallback_locales_path).first
+  def upload_locale_files(locale_files)
+    # We assert that the default locale contains all legitimate strings, and so
+    # we clean up orphaned content on PhraseApp post-upload by removing keys not
+    # in the default locale.
+    default_locale_index = locale_files.find_index { |f| f.locale_name == @default_locale }
+    raise RuntimeError.new("Missing default locale") unless default_locale_index
 
-    validate_files!([phraseapp_files_with_unverified, phraseapp_files_without_unverified, fallback_files])
+    upload_ids = @phraseapp_api.upload_files(locale_files, default_locale: @default_locale)
+    default_upload_id = upload_ids[default_locale_index]
 
-    phraseapp_files_with_unverified = phraseapp_files_with_unverified.index_by(&:name)
-    fallback_files                  = fallback_files.index_by(&:name)
-
-    # Clean empty strings from the data and merge the fallback data in:
-    # we want to replace unverified keys with their values in the fallback
-    phraseapp_files_without_unverified.map do |phraseapp_without_unverified_file|
-      without_unverified = clear_empty_strings!(phraseapp_without_unverified_file.parsed_content)
-      with_unverified    = clear_empty_strings!(phraseapp_files_with_unverified[phraseapp_without_unverified_file.name].parsed_content)
-      fallback           = clear_empty_strings!(fallback_files[phraseapp_without_unverified_file.name].parsed_content)
+    STDERR.puts "Removing keys not in default locale upload #{default_upload_id}"
+    @phraseapp_api.remove_keys_not_in_upload(default_upload_id)
+  end
 
-      restore_unverified_originals!(fallback, with_unverified, without_unverified)
-      @locale_file_class.from_hash(phraseapp_without_unverified_file.name, without_unverified)
-    end
+  def download_locale_files
+    known_locales = @phraseapp_api.fetch_locales
+    @phraseapp_api.download_files(known_locales, skip_unverified: false)
   end
 
-  private
+  def merge_locale_file(our_file, their_file, ancestor_file)
+    if our_file.nil?
+      their_file
+    elsif their_file.nil?
+      our_file
+    else
+      ancestor_file ||= empty_locale_file(our_file.locale_name)
 
-  def find_default_locale_file(locales, files)
-    default_locale = locales.find(&:default?)
+      resolved_content = Differ.resolve!(
+        original:  ancestor_file.parsed_content,
+        primary:   our_file.parsed_content,
+        secondary: their_file.parsed_content)
 
-    default_locale_file = files.find do |file|
-      file.name == default_locale.name
+      @locale_file_class.from_hash(our_file.locale_name, resolved_content)
     end
   end
 
-  def load_phraseapp_files(phraseapp_locales, skip_unverified)
-    @phraseapp_api.download_files(phraseapp_locales, skip_unverified)
+  def empty_locale_file(locale_name)
+    @locale_file_class.from_hash(locale_name, {})
   end
 
-  def load_locale_files(*paths)
-    loader = LocaleFile::Loader.new(@locale_file_class::EXTENSION)
-    paths.map do |path|
-      loader.filenames(path).map { |l| loader.load(l) }
+  def load_locale_files(*filenames)
+    filenames.map do |filename|
+      load_locale_file(filename)
     end
   end
 
-  def validate_files!(file_groups)
-    file_name_lists = file_groups.map do |files|
-      files.map(&:name).to_set
-    end
-
-    # If we don't have the exact same locales for all of the sources, we can't diff them
-    unless file_name_lists.uniq.size == 1
-      message = "Number of files differs. This tool does not yet support adding\
-      or removing langauges: #{file_name_lists}"
-      raise RuntimeError.new(message)
-    end
+  def load_local_file(filename)
+    @locale_file_class.load_file(filename)
   end
 
-  # Mutates without_verified to include the fallbacks where needed.
-  #
-  # For any keys in both `with_unverified` and `originals` but not present in
-  # `without_unverified`, restore the version from `originals` to
-  # `without_unverified`
-  def restore_unverified_originals!(fallback, with_unverified, without_unverified)
-    fallback.each do |key, value|
-      with_value = with_unverified[key]
-
-      case value
-      when Hash
-        if with_value.is_a?(Hash)
-          without_value = (without_unverified[key] ||= {})
-          restore_unverified_originals!(value, with_value, without_value)
-        end
-      else
-        if with_value && !with_value.is_a?(Hash) && !without_unverified.has_key?(key)
-          without_unverified[key] = value
-        end
-      end
+  def load_locale_directories(*paths)
+    paths.map do |path|
+      load_locale_directory(path)
     end
   end
 
-  def clear_empty_strings!(hash)
-    hash.delete_if do |key, value|
-      if value == ""
-        true
-      elsif value.is_a?(Hash)
-        clear_empty_strings!(value)
-        value.empty?
-      else
-        false
-      end
-    end
-    hash
+  def load_locale_directory(path)
+    @locale_file_class.load_directory(path)
   end
 
-  class LocaleFileUpdates
-    attr_reader :original_phraseapp_files, :resolved_files
+  def write_locale_file(path, locale_file)
+    File.write(path, locale_file.content)
+  end
 
-    def initialize(original_phraseapp_files, resolved_files)
-      @original_phraseapp_files = original_phraseapp_files
-      @resolved_files           = resolved_files
+  def write_locale_directory(path, locale_files)
+    locale_files.each do |locale_file|
+      full_path = File.join(path, locale_file.filename)
+      File.write(full_path, locale_file.content)
     end
+    STDERR.puts "Wrote #{locale_files.count} locale files to #{path}: #{locale_files.map(&:filename)}"
   end
 end
-

data/lib/phraseapp_updater/locale_file.rb

@@ -3,50 +3,91 @@ require "phraseapp_updater/locale_file/yaml_file"
 
 class PhraseAppUpdater
   class LocaleFile
-    attr_reader :name, :content, :parsed_content
+    attr_reader :locale_name, :content, :parsed_content
 
-    class BadFileTypeError < StandardError ; end
+    class BadFileTypeError < StandardError; end
 
-    def self.from_hash(name, hash)
-      raise RuntimeError.new("Must be implemented in a subclass.")
-    end
+    class << self
+      def from_hash(locale_name, hash)
+        new(locale_name, hash)
+      end
 
-    def self.class_for_file_format(type)
-      case type.downcase
-      when "json"
-        JSONFile
-      when "yml", "yaml"
-        YAMLFile
-      else
-        raise BadFileTypeError.new("Invalid file type: #{type}")
+      def from_file_content(locale_name, content)
+        new(locale_name, load(content))
       end
-    end
 
-    # Expects a Ruby hash
-    def initialize(name, content)
-      @name           = name
-      @content        = content
-      @parsed_content = parse(@content)
-      format_content!
+      def load_directory(directory)
+        Dir[File.join(directory, "*.#{extension}")].map do |filename|
+          load_file(filename)
+        end
+      end
+
+      def load_file(filename)
+        unless File.readable?(filename) && File.file?(filename)
+          raise RuntimeError.new("Couldn't read localization file at #{filename}")
+        end
+
+        locale_name = File.basename(filename).chomp(".#{extension}")
+        from_file_content(locale_name, File.read(filename))
+      end
+
+      private :new
+
+      def class_for_file_format(type)
+        case type.downcase
+        when "json"
+          JSONFile
+        when "yml", "yaml"
+          YAMLFile
+        else
+          raise BadFileTypeError.new("Invalid file type: #{type}")
+        end
+      end
+
+      def extension
+        raise RuntimeError.new("Abstract method")
+      end
+
+      def phraseapp_type
+        raise RuntimeError.new("Abstract method")
+      end
+
+      def load(_content)
+        raise RuntimeError.new("Abstract method")
+      end
+
+      def dump(_hash)
+        raise RuntimeError.new("Abstract method")
+      end
     end
 
     def to_s
-      "#{name}, #{content[0,20]}..."
+      locale_name
     end
 
-    def name_with_extension
-      "#{name}.#{self.class::EXTENSION}"
+    def filename
+      "#{locale_name}.#{self.class.extension}"
     end
 
     private
 
-    def parse(content)
-      raise RuntimeError.new("Must be implemented in a subclass.")
+    # Expects a Ruby hash
+    def initialize(locale_name, parsed_content)
+      @locale_name    = locale_name
+      @parsed_content = normalize_hash(parsed_content)
+      @content        = self.class.dump(@parsed_content)
+      freeze
     end
 
-    def format_content!
-      raise RuntimeError.new("Must be implemented in a subclass.")
+    def normalize_hash(hash)
+      hash.keys.sort_by(&:to_s).each_with_object({}) do |key, normalized_hash|
+        val = hash[key]
+        next if val == '' || (val.is_a?(Hash) && val.empty?)
+
+        val = normalize_hash(val) if val.is_a?(Hash)
+        key = key.to_s if key.is_a?(Symbol)
+        normalized_hash[key] = val
+      end
     end
   end
 end
-