phraseapp_updater 2.1.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7c95df3029f6b77ecd8ea760f7d0cac9003879acfc04b9ac5674e47ea41d2cb
-  data.tar.gz: 86f123de84997ba31604aa207ac51aa55e1ba9fc05758e2d89b4720d2be9d4e6
+  metadata.gz: f33deb3a77a37a47abdf44c354526cbe3ad3d10256bb2a046bf48922178187fd
+  data.tar.gz: f22f2ceed18653b09c24be4724707cdc40c39539535c226fd028bc03338437cd
 SHA512:
-  metadata.gz: 2c3e5ccb85014ec287d027b41146c142bc7e291c07c7a00184375381983c00ba62988539314f37e194aeddf2e3ab7588f985c18ed019023c2324483121f35812
-  data.tar.gz: 80bdb49e97c0e41d931b04689e6b95dac234975cff532937cc25c0237c55700703810f9146bed7a7cc26583f24a1de38a910f8339fc4a2f12bf4e8ee3bcf57fb
+  metadata.gz: 822cc67247e2e2752397127f6b4e5e0d893cdfc3c64a29f14067d2bc5b6e131f3924d5cad982960df65da4b47971c12ad43fc9caf9c9ff15e756a596a64c163e
+  data.tar.gz: a2e1152e1a376690722722d73c2654d4ab4e417530b66b86f7562cd0b17e54f06976a295bae601ec0d4ed1e8570a518eef9a37ff317f24001643e711c0365e6c

data/bin/phraseapp_common.sh

@@ -1,16 +1,22 @@
 #!/usr/bin/env bash
 
 # Set up a working directory
-working_directory=$(mktemp -d -t phraseapp.XXXXXX)
+working_directory=$(mktemp -d phraseapp.XXXXXX)
 
 function cleanup_working_directory(){
-    rm -rf "${working_directory}"
+  local now archive
+  now="$(date "+%Y%m%d%H%M%S")"
+  archive="/tmp/phraseapp-updater-$now.tar.gz"
+  tar -C "$(dirname "$working_directory")" -czf "$archive" "$(basename "${working_directory}")"
+  rm -rf "${working_directory}"
+  echo "Working files saved to $archive"
 }
 
 trap "cleanup_working_directory" EXIT SIGINT
 
 function make_temporary_directory() {
-    mktemp -d "${working_directory}/XXXXXXXX"
+    name="${1:-tmp}"
+    mktemp -d "${working_directory}/$name.XXXXXXXX"
 }
 
 function make_tree_from_directory() {
@@ -35,15 +41,15 @@ function make_tree_from_directory() {
 }
 
 
-function extract_commit() {
-    extract_files "$1:${PREFIX}"
+function extract_prefix_from_commit() {
+    extract_files "$1" "$2:${PREFIX}"
 }
 
 function extract_files() {
     local path
-    path=$(make_temporary_directory)
+    path=$(make_temporary_directory "git.${1}.${2%%:*}")
 
-    git archive --format=tar "$1" | tar -x -C "${path}"
+    git archive --format=tar "$2" | tar -x -C "${path}"
 
     echo "${path}"
 }

data/bin/synchronize_phraseapp.sh

@@ -34,7 +34,7 @@ if [ "$local_branch" ] && [ "$current_branch" != "$local_branch" ]; then
 fi
 
 # First, fetch the current contents of PhraseApp's staged ("verified") state.
-current_phraseapp_path=$(make_temporary_directory)
+current_phraseapp_path=$(make_temporary_directory phraseapp_download)
 common_ancestor=$(phraseapp_updater download "${current_phraseapp_path}" \
                       --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
                       --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
@@ -61,8 +61,8 @@ elif ! git merge-base --is-ancestor "${common_ancestor}" "${current_branch}"; th
     skip_ancestor_merge=t
 fi
 
-current_branch_path=$(extract_commit "${current_branch}")
-common_ancestor_path=$(extract_commit "${common_ancestor}")
+current_branch_path=$(extract_prefix_from_commit current_branch "${current_branch}")
+common_ancestor_path=$(extract_prefix_from_commit common_ancestor "${common_ancestor}")
 
 current_phraseapp_tree=$(make_tree_from_directory "${current_phraseapp_path}")
 
@@ -87,7 +87,7 @@ if [ "${phraseapp_changed}" = 't' ] && [ "${branch_changed}" = 't' ]; then
     echo "$BRANCH branch and PhraseApp both changed: 3-way merging" >&2
 
     # 3-way merge
-    merge_resolution_path=$(make_temporary_directory)
+    merge_resolution_path=$(make_temporary_directory merge_resolution)
     phraseapp_updater merge "${common_ancestor_path}" "${current_branch_path}" "${current_phraseapp_path}" \
                       --to "${merge_resolution_path}" \
                       --verbose="${VERBOSE}"          \

data/lib/phraseapp_updater/phraseapp_api.rb

@@ -2,7 +2,8 @@
 
 require 'phraseapp_updater/locale_file'
 require 'phraseapp_updater/index_by'
-require 'phraseapp-ruby'
+require 'uri'
+require 'phrase'
 require 'parallel'
 require 'tempfile'
 
@@ -13,16 +14,24 @@ class PhraseAppUpdater
     PAGE_SIZE = 100
 
     def initialize(api_key, project_id, locale_file_class)
-      @client            = PhraseApp::Client.new(PhraseApp::Auth::Credentials.new(token: api_key))
+      config = Phrase::Configuration.new do |c|
+        c.api_key['Authorization'] = api_key
+        c.api_key_prefix['Authorization'] = 'token'
+        c.debugging = false
+      end
+
+      @client            = Phrase::ApiClient.new(config)
       @project_id        = project_id
       @locale_file_class = locale_file_class
     end
 
     def create_project(name, parent_commit)
-      params = PhraseApp::RequestParams::ProjectParams.new(
+      params = Phrase::ProjectCreateParameters.new(
         name: name,
         main_format: @locale_file_class.phraseapp_type)
-      project = phraseapp_request { @client.project_create(params) }
+
+      project = phraseapp_request(Phrase::ProjectsApi, :project_create, params)
+
       STDERR.puts "Created project #{name} for #{parent_commit}"
 
       @project_id = project.id
@@ -32,7 +41,8 @@ class PhraseAppUpdater
     end
 
     def lookup_project_id(name)
-      result, = paginate_request(:projects_list, limit: 1) { |p| p.name == name }
+      result, = paginated_request(Phrase::ProjectsApi, :projects_list, { per_page: PAGE_SIZE }, limit: 1) { |p| p.name == name }
+
       raise ProjectNotFoundError.new(name) if result.nil?
 
       result.id
@@ -41,9 +51,10 @@ class PhraseAppUpdater
     # We mark projects with their parent git commit using a tag with a
     # well-known prefix. We only allow one tag with this prefix at once.
     def read_parent_commit
-      git_tag, = paginate_request(:tags_list, @project_id, limit: 1) do |t|
+      git_tag, = paginated_request(Phrase::TagsApi, :tags_list, @project_id, limit: 1) do |t|
         t.name.start_with?(GIT_TAG_PREFIX)
       end
+
       raise MissingGitParentError.new if git_tag.nil?
 
       git_tag.name.delete_prefix(GIT_TAG_PREFIX)
@@ -51,35 +62,28 @@ class PhraseAppUpdater
 
     def update_parent_commit(commit_hash)
       previous_parent = read_parent_commit
-      phraseapp_request do
-        @client.tag_delete(@project_id, GIT_TAG_PREFIX + previous_parent)
-      end
+      phraseapp_request(Phrase::TagsApi, :tag_delete, @project_id, GIT_TAG_PREFIX + previous_parent)
       store_parent_commit(commit_hash)
     end
 
     def fetch_locales
-      # This is a paginated API, however the maximum page size of 100 is well
-      # above our expected locale size, so we take the first page only for now
-      phraseapp_request { @client.locales_list(@project_id, 1, 100) }.map do |pa_locale|
-        Locale.new(pa_locale)
-      end
+      locales = paginated_request(Phrase::LocalesApi, :locales_list, @project_id)
+      locales.map { |pa_locale| Locale.new(pa_locale) }
     end
 
     def create_locale(name, default: false)
-      phraseapp_request do
-        params = PhraseApp::RequestParams::LocaleParams.new(
-          name: name,
-          code: name,
-          default: default,
-        )
-        @client.locale_create(@project_id, params)
-      end
+      params = Phrase::LocaleCreateParameters.new(
+        name: name,
+        code: name,
+        default: default,
+      )
+      phraseapp_request(Phrase::LocalesApi, :locale_create, @project_id, params)
     end
 
     def download_files(locales, skip_unverified:)
       results = threaded_request(locales) do |locale|
         STDERR.puts "Downloading file for #{locale}"
-        download_file(locale, skip_unverified)
+        download_locale(locale, skip_unverified)
       end
 
       locales.zip(results).map do |locale, file_contents|
@@ -87,18 +91,65 @@ class PhraseAppUpdater
       end
     end
 
+    # Empirically, PhraseApp fails to parse the uploaded files when uploaded in
+    # parallel. Give it a better chance by uploading them one at a time.
     def upload_files(locale_files, default_locale:)
-      known_locales = fetch_locales.index_by(&:name)
+      is_default = ->(l) { l.locale_name == default_locale }
 
+      # Ensure the locales all exist
+      STDERR.puts('Creating locales')
+      known_locales = fetch_locales.index_by(&:name)
       threaded_request(locale_files) do |locale_file|
         unless known_locales.has_key?(locale_file.locale_name)
-          create_locale(locale_file.locale_name,
-                        default: (locale_file.locale_name == default_locale))
+          create_locale(locale_file.locale_name, default: is_default.(locale_file))
+        end
+      end
+
+      # Upload the files in a stable order, ensuring the default locale is first.
+      locale_files.sort! do |a, b|
+        next -1 if is_default.(a)
+        next 1  if is_default.(b)
+
+        a.locale_name <=> b.locale_name
+      end
+
+      uploads = {}
+
+      uploads = locale_files.to_h do |locale_file|
+        STDERR.puts("Uploading #{locale_file}")
+        upload_id = upload_file(locale_file)
+        [upload_id, locale_file]
+      end
+
+      # Validate the uploads, retrying failures as necessary
+      successful_upload_ids = {}
+
+      STDERR.puts('Verifying uploads...')
+      until uploads.empty?
+        threaded_request(uploads.to_a) do |upload_id, locale_file|
+          upload = phraseapp_request(Phrase::UploadsApi, :upload_show, @project_id, upload_id)
+
+          case upload.state
+          when "enqueued", "processing"
+            STDERR.puts("#{locale_file}: still processing")
+          when "success"
+            STDERR.puts("#{locale_file}: success")
+            successful_upload_ids[locale_file.locale_name] = upload_id
+            uploads.delete(upload_id)
+          when "error"
+            STDERR.puts("#{locale_file}: upload failure, retrying")
+            new_upload_id = upload_file(locale_file)
+            uploads.delete(upload_id)
+            uploads[new_upload_id] = locale_file
+          else
+            raise RuntimeError.new("Unknown upload state: #{upload.state}")
+          end
         end
 
-        STDERR.puts "Uploading #{locale_file}"
-        upload_file(locale_file)
+        sleep(2) unless uploads.empty?
       end
+
+      successful_upload_ids
     end
 
     def remove_keys_not_in_uploads(upload_ids)
@@ -108,105 +159,101 @@ class PhraseAppUpdater
       end
     end
 
-    def download_file(locale, skip_unverified)
-      download_params = PhraseApp::RequestParams::LocaleDownloadParams.new
-
-      download_params.file_format                  = @locale_file_class.phraseapp_type
-      download_params.skip_unverified_translations = skip_unverified
+    def download_locale(locale, skip_unverified)
+      opts = {
+        file_format: @locale_file_class.phraseapp_type,
+        skip_unverified_translations: skip_unverified,
+      }
 
-      phraseapp_request { @client.locale_download(@project_id, locale.id, download_params) }
+      # Avoid allocating a tempfile (and emitting unnecessary warnings) by using `return_type` of `String`
+      phraseapp_request(Phrase::LocalesApi, :locale_download, @project_id, locale.id, return_type: 'String', **opts)
     end
 
     def upload_file(locale_file)
-      upload_params = create_upload_params(locale_file.locale_name)
-
       # The PhraseApp gem only accepts a filename to upload,
       # so we need to write the file out and pass it the path
       Tempfile.create([locale_file.locale_name, ".json"]) do |f|
         f.write(locale_file.content)
         f.close
 
-        upload_params.file = f.path
-        phraseapp_request { @client.upload_create(@project_id, upload_params) }.id
+        opts = {
+          file:                 f,
+          file_encoding:        'UTF-8',
+          file_format:          @locale_file_class.phraseapp_type,
+          locale_id:            locale_file.locale_name,
+          skip_unverification:  false,
+          update_translations:  true,
+          tags:                 [generate_upload_tag],
+        }
+
+        result = phraseapp_request(Phrase::UploadsApi, :upload_create, @project_id, **opts)
+
+        result.id
       end
     end
 
     def remove_keys_not_in_upload(upload_id)
-      delete_params   = PhraseApp::RequestParams::KeysDeleteParams.new
-      delete_params.q = "unmentioned_in_upload:#{upload_id}"
-
-      begin
-        phraseapp_request { @client.keys_delete(@project_id, delete_params) }
-      rescue RuntimeError => _e
-        # PhraseApp will accept but mark invalid uploads, however the gem
-        # returns the same response in both cases. If we call this API
-        # with the ID of an upload of a bad file, it will fail.
-        # This usually occurs when sending up an empty file, which is
-        # a case we can ignore. However, it'd be better to have a way
-        # to detect a bad upload and find the cause.
-      end
+      delete_pattern = "unmentioned_in_upload:#{upload_id}"
+      phraseapp_request(Phrase::KeysApi, :keys_delete_collection, @project_id, q: delete_pattern)
     end
 
     private
 
     def store_parent_commit(commit_hash)
-      params = PhraseApp::RequestParams::TagParams.new(
-        name: GIT_TAG_PREFIX + commit_hash)
-      phraseapp_request { @client.tag_create(@project_id, params) }
+      params = Phrase::TagCreateParameters.new(name: GIT_TAG_PREFIX + commit_hash)
+      phraseapp_request(Phrase::TagsApi,:tag_create, @project_id, params)
+    end
+
+    def wrap_phrase_errors
+      yield
+    rescue Phrase::ApiError => e
+      if e.code == 401
+        raise BadAPIKeyError.new(e)
+      elsif e.message.match?(/not found/)
+        raise BadProjectIDError.new(e, @project_id)
+      elsif e.message.match?(/has already been taken/)
+        raise ProjectNameTakenError.new(e)
+      else
+        raise
+      end
     end
 
-    def paginate_request(method, *params, limit: nil, &filter)
+    def paginated_request(api_class, method, *params, limit: nil, **opts, &filter)
+
+      api_instance = api_class.new(@client)
       page = 1
       results = []
 
       loop do
-        response = phraseapp_request do
-          @client.public_send(method, *params, page, PAGE_SIZE)
+        response = wrap_phrase_errors do
+          api_instance.public_send(method, *params, opts.merge(page: page, page_size: PAGE_SIZE))
         end
 
-        break if response.empty?
+        break if response.data.empty?
 
-        matches = response.filter(&filter)
+        matches = response.data
+        matches = matches.filter(&filter) if filter
         matches = matches[0, limit - results.size] if limit
 
-        unless matches.empty?
-          results.concat(matches)
+        results.concat(matches) unless matches.empty?
 
-          break if results.size == limit
-        end
+        break if results.size == limit
+        break unless response.next_page?
 
-        page += 1
+        page = response.next_page
       end
 
       results
     end
 
-    def phraseapp_request(&block)
-      res, err = block.call
+    def phraseapp_request(api_class, method, *params, **opts)
+      api_instance = api_class.new(@client)
 
-      unless err.nil?
-        error =
-          if err.respond_to?(:error)
-            err.error
-          else
-            err.errors.join('|')
-          end
-
-        raise RuntimeError.new(error)
+      response = wrap_phrase_errors do
+        api_instance.public_send(method, *params, opts)
       end
 
-      res
-
-    rescue RuntimeError => e
-      if e.message.match?(/\(401\)/)
-        raise BadAPIKeyError.new(e)
-      elsif e.message.match?(/not found/)
-        raise BadProjectIDError.new(e, @project_id)
-      elsif e.message.match?(/has already been taken/)
-        raise ProjectNameTakenError.new(e)
-      else
-        raise
-      end
+      response.data
     end
 
     # PhraseApp allows two concurrent connections at a time.
@@ -216,17 +263,6 @@ class PhraseAppUpdater
       Parallel.map(worklist, in_threads: THREAD_COUNT, &block)
     end
 
-    def create_upload_params(locale_name)
-      upload_params = PhraseApp::RequestParams::UploadParams.new
-      upload_params.file_encoding       = 'UTF-8'
-      upload_params.file_format         = @locale_file_class.phraseapp_type
-      upload_params.locale_id           = locale_name
-      upload_params.skip_unverification = false
-      upload_params.update_translations = true
-      upload_params.tags                = [generate_upload_tag]
-      upload_params
-    end
-
     def generate_upload_tag
       "phraseapp_updater_upload_#{Time.now.strftime('%Y%m%d%H%M%S')}"
     end

data/lib/phraseapp_updater/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class PhraseAppUpdater
-  VERSION = '2.1.1'
+  VERSION = '3.0.0'
 end

data/lib/phraseapp_updater.rb

@@ -102,13 +102,14 @@ class PhraseAppUpdater
     # We assert that the default locale contains all legitimate strings, and so
     # we clean up orphaned content on PhraseApp post-upload by removing keys not
     # in the default locale.
-    default_locale_index = locale_files.find_index { |f| f.locale_name == @default_locale }
-    raise RuntimeError.new("Missing default locale") unless default_locale_index
+    unless locale_files.find { |f| f.locale_name == @default_locale }
+      raise RuntimeError.new("Missing default locale")
+    end
 
     upload_ids = @phraseapp_api.upload_files(locale_files, default_locale: @default_locale)
-    default_upload_id = upload_ids[default_locale_index]
+    default_upload_id = upload_ids.fetch(@default_locale)
 
-    STDERR.puts "Removing keys not in default locale upload #{default_upload_id}"
+    STDERR.puts "Removing keys not in default locale '#{@default_locale}' upload '#{default_upload_id}'"
     @phraseapp_api.remove_keys_not_in_upload(default_upload_id)
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: phraseapp_updater
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 3.0.0
 platform: ruby
 authors:
-- Kevin Griffin
+- iKnow Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2023-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -25,33 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.19'
 - !ruby/object:Gem::Dependency
-  name: phraseapp-ruby
+  name: phrase
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.8.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.8.3
 - !ruby/object:Gem::Dependency
   name: hashdiff
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -167,22 +167,13 @@ dependencies:
 description: A tool for merging data on PhraseApp with local changes (usually two
   git revisions)
 email:
-- kev@bibo.com.ph
+- systems@iknow.jp
 executables:
 - phraseapp_updater
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
-- ".envrc"
-- ".gitignore"
-- ".rspec"
-- ".travis.yml"
-- CODE_OF_CONDUCT.markdown
-- Gemfile
 - LICENSE.txt
-- README.markdown
-- Rakefile
 - bin/console
 - bin/phraseapp_common.sh
 - bin/phraseapp_updater
@@ -197,13 +188,7 @@ files:
 - lib/phraseapp_updater/phraseapp_api.rb
 - lib/phraseapp_updater/version.rb
 - lib/phraseapp_updater/yml_config_loader.rb
-- nix/gem/Gemfile
-- nix/gem/Gemfile.lock
-- nix/gem/gemset.nix
-- nix/generate.rb
-- phraseapp_updater.gemspec
-- shell.nix
-homepage: https://app.engoo.com
+homepage: https://github.com/iknow/phraseapp_updater
 licenses:
 - MIT
 metadata: {}
@@ -222,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: A three-way differ for PhraseApp projects.