phraseapp_updater 0.1.7 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,141 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+ set -o pipefail
5
+
6
+ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
7
+ . "${SCRIPTPATH}/phraseapp_common.sh"
8
+
9
+ if [ ! -d ".git" ]; then
10
+ echo "Error: must be run in a git checkout root" >&2
11
+ exit 1
12
+ fi
13
+
14
+ # Configuration is via environment, and expected to be provided from a Ruby
15
+ # driver.
16
+ for variable in PHRASEAPP_API_KEY PHRASEAPP_PROJECT_ID BRANCH REMOTE PREFIX FILE_FORMAT NO_COMMIT; do
17
+ if [ -z "${!variable}" ]; then
18
+ echo "Error: must specify $variable" >&2
19
+ exit 1
20
+ fi
21
+ done
22
+
23
+ # Ensure we're up to date
24
+ git fetch "${REMOTE}"
25
+
26
+ current_branch=$(git rev-parse "${REMOTE}/${BRANCH}")
27
+
28
+ # First, fetch the current contents of PhraseApp's staged ("verified") state.
29
+ current_phraseapp_path=$(make_temporary_directory)
30
+ common_ancestor=$(phraseapp_updater download "${current_phraseapp_path}" \
31
+ --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
32
+ --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
33
+ --file_format="${FILE_FORMAT}")
34
+
35
+ # If common_ancestor is not available or reachable from BRANCH, we've been
36
+ # really naughty and rebased without uploading the result to PhraseApp
37
+ # afterwards. If it's not available, we lose: the best we can do is manually
38
+ # perform a 2-way diff and force upload to phraseapp. If it's still available
39
+ # but not reachable, we can still try and perform a 3 way merge, but the results
40
+ # will not be as accurate: warn the user.
41
+ if ! git cat-file -e "${common_ancestor}^{commit}"; then
42
+ echo "Common ancestor commit could not be found: was '${BRANCH}' rebased without updating PhraseApp?" >&2
43
+ exit 1
44
+ elif ! git merge-base --is-ancestor "${common_ancestor}" "${current_branch}"; then
45
+ echo "Warning: ancestor commit was not reachable from '${BRANCH}': 3-way merge may be inaccurate" >&2
46
+ fi
47
+
48
+ current_branch_path=$(extract_commit "${current_branch}")
49
+ common_ancestor_path=$(extract_commit "${common_ancestor}")
50
+
51
+ current_phraseapp_tree=$(make_tree_from_directory "${current_phraseapp_path}")
52
+
53
+ # We have four cases to handle:
54
+ # 1: PhraseApp and BRANCH locales both changed since common_ancestor:
55
+ # * Record current PhraseApp in a commit A with parent `common_ancestor`
56
+ # * 3-way merge BRANCH locales and PhraseApp contents
57
+ # * Commit the result to BRANCH with parents BRANCH and A, yielding B
58
+ # * Push the result to phraseapp with new common_ancestor B
59
+ # 2: Only BRANCH changed since common_ancestor:
60
+ # * Push BRANCH locales to PhraseApp with new common_ancestor BRANCH
61
+ # 3: Only PhraseApp changed since common_ancestor:
62
+ # * Commit and push PhraseApp contents to BRANCH yielding commit A
63
+ # * Update PhraseApp common_ancestor to A
64
+ # 4: Neither changed:
65
+ # * Do nothing
66
+
67
+ phraseapp_changed=$(if locales_changed "${common_ancestor_path}" "${current_phraseapp_path}"; then echo t; else echo f; fi)
68
+ branch_changed=$(if locales_changed "${common_ancestor_path}" "${current_branch_path}"; then echo t; else echo f; fi)
69
+
70
+ if [ "${phraseapp_changed}" = 't' ] && [ "${branch_changed}" = 't' ]; then
71
+ echo "$BRANCH branch and PhraseApp both changed: 3-way merging" >&2
72
+
73
+ # 3-way merge
74
+ merge_resolution_path=$(make_temporary_directory)
75
+ phraseapp_updater merge "${common_ancestor_path}" "${current_branch_path}" "${current_phraseapp_path}" \
76
+ --to "${merge_resolution_path}" \
77
+ --file_format="${FILE_FORMAT}"
78
+
79
+ if [ "$NO_COMMIT" != 't' ]; then
80
+ # Create a commit to record the pre-merge state of PhraseApp
81
+ phraseapp_commit=$(git commit-tree "${current_phraseapp_tree}" \
82
+ -p "${common_ancestor}" \
83
+ -m "Remote locale changes made on PhraseApp (drop when rebasing)")
84
+
85
+ # Commit merge result to PREFIX in BRANCH
86
+ merge_resolution_tree=$(make_tree_from_directory "${merge_resolution_path}")
87
+ merged_branch_tree=$(replace_nested_tree "${current_branch}^{tree}" "${PREFIX}" "${merge_resolution_tree}")
88
+
89
+ merge_commit=$(git commit-tree "${merged_branch_tree}" \
90
+ -p "${current_branch}" \
91
+ -p "${phraseapp_commit}" \
92
+ -m "Merged locale changes from PhraseApp")
93
+
94
+ # Push to BRANCH
95
+ git push "${REMOTE}" "${merge_commit}:refs/heads/${BRANCH}"
96
+ new_parent_commit="${merge_commit}"
97
+ else
98
+ # Merge is only to phraseapp: record current branch as new common ancestor
99
+ echo "Not committing to $BRANCH" >&2
100
+ new_parent_commit="${current_branch}"
101
+ fi
102
+
103
+ # Push merge result to phraseapp
104
+ phraseapp_updater upload "${merge_resolution_path}" \
105
+ --parent_commit="${new_parent_commit}" \
106
+ --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
107
+ --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
108
+ --file_format="${FILE_FORMAT}"
109
+
110
+ elif [ "${branch_changed}" = 't' ]; then
111
+ echo "Only $BRANCH branch changed: updating PhraseApp" >&2
112
+
113
+ # Upload to phraseapp
114
+ phraseapp_updater upload "${current_branch_path}" \
115
+ --parent_commit="${current_branch}" \
116
+ --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
117
+ --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}" \
118
+ --file_format="${FILE_FORMAT}"
119
+
120
+ elif [ "${phraseapp_changed}" = 't' ]; then
121
+ if [ "$NO_COMMIT" != 't' ]; then
122
+ echo "Only PhraseApp changed: updating $BRANCH branch" >&2
123
+
124
+ updated_branch_tree=$(replace_nested_tree "${current_branch}^{tree}" "${PREFIX}" "${current_phraseapp_tree}")
125
+ update_commit=$(git commit-tree "${updated_branch_tree}" \
126
+ -p "${current_branch}" \
127
+ -m "Incorporate locale changes from PhraseApp")
128
+
129
+ git push "${REMOTE}" "${update_commit}:refs/heads/${BRANCH}"
130
+
131
+ # Set ancestor on phraseapp
132
+ phraseapp_updater update_parent_commit \
133
+ --parent_commit="${update_commit}" \
134
+ --phraseapp_api_key="${PHRASEAPP_API_KEY}" \
135
+ --phraseapp_project_id="${PHRASEAPP_PROJECT_ID}"
136
+ else
137
+ echo "Only PhraseApp changed: not committing to $BRANCH branch" >&2
138
+ fi
139
+ else
140
+ echo "No changes made since common ancestor" >&2
141
+ fi
@@ -1,176 +1,165 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'phraseapp_updater/version'
2
4
  require 'phraseapp_updater/index_by'
3
5
  require 'phraseapp_updater/differ'
4
6
  require 'phraseapp_updater/locale_file'
5
- require 'phraseapp_updater/locale_file/loader'
6
7
  require 'phraseapp_updater/phraseapp_api'
7
8
  require 'phraseapp_updater/yml_config_loader'
8
9
 
9
10
  class PhraseAppUpdater
10
11
  using IndexBy
11
12
 
12
- def initialize(phraseapp_api_key, phraseapp_project_id, file_format)
13
+ def self.for_new_project(phraseapp_api_key, phraseapp_project_name, file_format, parent_commit)
14
+ api = PhraseAppAPI.new(phraseapp_api_key, nil, LocaleFile.class_for_file_format(file_format))
15
+ project_id = api.create_project(phraseapp_project_name, parent_commit)
16
+ return self.new(phraseapp_api_key, project_id, file_format), project_id
17
+ end
18
+
19
+ def initialize(phraseapp_api_key, phraseapp_project_id, file_format, default_locale: 'en')
13
20
  @locale_file_class = LocaleFile.class_for_file_format(file_format)
21
+ @default_locale = default_locale
14
22
  @phraseapp_api = PhraseAppAPI.new(phraseapp_api_key, phraseapp_project_id, @locale_file_class)
15
23
  end
16
24
 
17
- def self.load_config(config_file_path)
18
- YMLConfigLoader.new(config_file_path)
25
+ def diff_directories(our_path, their_path)
26
+ (our_locales, their_locales) = load_locale_directories(our_path, their_path)
27
+ diff_locale_files(our_locales, their_locales)
19
28
  end
20
29
 
21
- def push(previous_locales_path, new_locales_path)
22
- phraseapp_locales = @phraseapp_api.download_locales
30
+ def merge_directories(our_path, their_path, ancestor_path, result_path)
31
+ (our_locales, their_locales, ancestor_locales) =
32
+ load_locale_directories(our_path, their_path, ancestor_path)
23
33
 
24
- phraseapp_files = load_phraseapp_files(phraseapp_locales, false)
34
+ merged_locales = merge_locale_files(our_locales, their_locales, ancestor_locales)
25
35
 
26
- (previous_locale_files, new_locale_files) =
27
- load_locale_files(previous_locales_path, new_locales_path)
36
+ write_locale_directory(result_path, merged_locales)
37
+ end
28
38
 
29
- validate_files!([phraseapp_files, previous_locale_files, new_locale_files])
39
+ def merge_files(ours, theirs, ancestor, result)
40
+ our_file, their_file = load_locale_files(ours, theirs)
41
+ # Read the ancestor if provided
42
+ ancestor_file = load_locale_file(ancestor) unless ancestor.nil?
30
43
 
31
- new_locale_files = new_locale_files.index_by(&:name)
32
- phraseapp_files = phraseapp_files.index_by(&:name)
44
+ result_file = merge_locale_files(our_file, their_file, ancestor_file)
33
45
 
34
- resolved_files = previous_locale_files.map do |previous_locale_file|
35
- new_locale_file = new_locale_files.fetch(previous_locale_file.name)
36
- phraseapp_file = phraseapp_files.fetch(previous_locale_file.name)
46
+ write_locale_file(result, result_file)
47
+ end
37
48
 
38
- resolved_content = Differ.resolve!(original: previous_locale_file.parsed_content,
39
- primary: new_locale_file.parsed_content,
40
- secondary: phraseapp_file.parsed_content)
49
+ def upload_directory(path)
50
+ locales = load_locale_directory(path)
51
+ upload_locale_files(locales)
52
+ end
41
53
 
42
- @locale_file_class.from_hash(previous_locale_file.name, resolved_content)
43
- end
54
+ def download_to_directory(path)
55
+ locale_files = download_locale_files
56
+ write_locale_directory(path, locale_files)
57
+ end
44
58
 
45
- # Upload all of the secondary languages first,
46
- # so that the missing keys in them get filled in
47
- # with blanks on PhraseApp by the default locale.
48
- # If we do the clean up after uploading the default
49
- # locale file, these blanks will get cleaned because
50
- # they're not mentioned in the secondary locale files.
59
+ def update_parent_commit(parent_commit)
60
+ @phraseapp_api.update_parent_commit(parent_commit)
61
+ end
51
62
 
52
- default_locale_file = find_default_locale_file(phraseapp_locales, resolved_files)
63
+ def read_parent_commit
64
+ @phraseapp_api.read_parent_commit
65
+ end
53
66
 
54
- resolved_files.delete(default_locale_file)
67
+ private
55
68
 
56
- changed_files = resolved_files.select do |file|
57
- file.parsed_content != phraseapp_files[file.name].parsed_content
69
+ def diff_locale_files(our_locales, their_locales)
70
+ (our_content, their_content) = [our_locales, their_locales].map do |locales|
71
+ locales.each_with_object({}) do |locale, h|
72
+ h[locale.locale_name] = locale.parsed_content
73
+ end
58
74
  end
59
75
 
60
- upload_ids = @phraseapp_api.upload_files(changed_files)
61
- @phraseapp_api.remove_keys_not_in_uploads(upload_ids)
76
+ HashDiff.diff(our_content, their_content)
77
+ end
62
78
 
63
- puts "Uploading #{default_locale_file}"
64
- upload_id = @phraseapp_api.upload_file(default_locale_file)
79
+ def merge_locale_files(our_locales, their_locales, ancestor_locales)
80
+ ours = our_locales.index_by(&:locale_name)
81
+ theirs = their_locales.index_by(&:locale_name)
82
+ ancestors = ancestor_locales.index_by(&:locale_name)
65
83
 
66
- puts "Removing keys not in upload #{upload_id}"
67
- @phraseapp_api.remove_keys_not_in_upload(upload_id)
84
+ locale_names = Set.new.merge(ours.keys).merge(theirs.keys)
68
85
 
69
- LocaleFileUpdates.new(phraseapp_files.values, [default_locale_file] + resolved_files)
86
+ locale_names.map do |locale_name|
87
+ our_file = ours.fetch(locale_name)
88
+ their_file = theirs.fetch(locale_name)
89
+ ancestor_file = ancestors[locale_name]
90
+ merge_locale_file(our_file, their_file, ancestor_file)
91
+ end
70
92
  end
71
93
 
72
- def pull(fallback_locales_path)
73
- phraseapp_locales = @phraseapp_api.download_locales
74
-
75
- phraseapp_files_without_unverified = load_phraseapp_files(phraseapp_locales, true)
76
- phraseapp_files_with_unverified = load_phraseapp_files(phraseapp_locales, false)
77
- fallback_files = load_locale_files(fallback_locales_path).first
94
+ def upload_locale_files(locale_files)
95
+ # We assert that the default locale contains all legitimate strings, and so
96
+ # we clean up orphaned content on PhraseApp post-upload by removing keys not
97
+ # in the default locale.
98
+ default_locale_index = locale_files.find_index { |f| f.locale_name == @default_locale }
99
+ raise RuntimeError.new("Missing default locale") unless default_locale_index
78
100
 
79
- validate_files!([phraseapp_files_with_unverified, phraseapp_files_without_unverified, fallback_files])
101
+ upload_ids = @phraseapp_api.upload_files(locale_files, default_locale: @default_locale)
102
+ default_upload_id = upload_ids[default_locale_index]
80
103
 
81
- phraseapp_files_with_unverified = phraseapp_files_with_unverified.index_by(&:name)
82
- fallback_files = fallback_files.index_by(&:name)
83
-
84
- # Clean empty strings from the data and merge the fallback data in:
85
- # we want to replace unverified keys with their values in the fallback
86
- phraseapp_files_without_unverified.map do |phraseapp_without_unverified_file|
87
- without_unverified = clear_empty_strings!(phraseapp_without_unverified_file.parsed_content)
88
- with_unverified = clear_empty_strings!(phraseapp_files_with_unverified[phraseapp_without_unverified_file.name].parsed_content)
89
- fallback = clear_empty_strings!(fallback_files[phraseapp_without_unverified_file.name].parsed_content)
104
+ STDERR.puts "Removing keys not in default locale upload #{default_upload_id}"
105
+ @phraseapp_api.remove_keys_not_in_upload(default_upload_id)
106
+ end
90
107
 
91
- restore_unverified_originals!(fallback, with_unverified, without_unverified)
92
- @locale_file_class.from_hash(phraseapp_without_unverified_file.name, without_unverified)
93
- end
108
+ def download_locale_files
109
+ known_locales = @phraseapp_api.fetch_locales
110
+ @phraseapp_api.download_files(known_locales, skip_unverified: false)
94
111
  end
95
112
 
96
- private
113
+ def merge_locale_file(our_file, their_file, ancestor_file)
114
+ if our_file.nil?
115
+ their_file
116
+ elsif their_file.nil?
117
+ our_file
118
+ else
119
+ ancestor_file ||= empty_locale_file(our_file.locale_name)
97
120
 
98
- def find_default_locale_file(locales, files)
99
- default_locale = locales.find(&:default?)
121
+ resolved_content = Differ.resolve!(
122
+ original: ancestor_file.parsed_content,
123
+ primary: our_file.parsed_content,
124
+ secondary: their_file.parsed_content)
100
125
 
101
- default_locale_file = files.find do |file|
102
- file.name == default_locale.name
126
+ @locale_file_class.from_hash(our_file.locale_name, resolved_content)
103
127
  end
104
128
  end
105
129
 
106
- def load_phraseapp_files(phraseapp_locales, skip_unverified)
107
- @phraseapp_api.download_files(phraseapp_locales, skip_unverified)
130
+ def empty_locale_file(locale_name)
131
+ @locale_file_class.from_hash(locale_name, {})
108
132
  end
109
133
 
110
- def load_locale_files(*paths)
111
- loader = LocaleFile::Loader.new(@locale_file_class::EXTENSION)
112
- paths.map do |path|
113
- loader.filenames(path).map { |l| loader.load(l) }
134
+ def load_locale_files(*filenames)
135
+ filenames.map do |filename|
136
+ load_locale_file(filename)
114
137
  end
115
138
  end
116
139
 
117
- def validate_files!(file_groups)
118
- file_name_lists = file_groups.map do |files|
119
- files.map(&:name).to_set
120
- end
121
-
122
- # If we don't have the exact same locales for all of the sources, we can't diff them
123
- unless file_name_lists.uniq.size == 1
124
- message = "Number of files differs. This tool does not yet support adding\
125
- or removing langauges: #{file_name_lists}"
126
- raise RuntimeError.new(message)
127
- end
140
+ def load_local_file(filename)
141
+ @locale_file_class.load_file(filename)
128
142
  end
129
143
 
130
- # Mutates without_verified to include the fallbacks where needed.
131
- #
132
- # For any keys in both `with_unverified` and `originals` but not present in
133
- # `without_unverified`, restore the version from `originals` to
134
- # `without_unverified`
135
- def restore_unverified_originals!(fallback, with_unverified, without_unverified)
136
- fallback.each do |key, value|
137
- with_value = with_unverified[key]
138
-
139
- case value
140
- when Hash
141
- if with_value.is_a?(Hash)
142
- without_value = (without_unverified[key] ||= {})
143
- restore_unverified_originals!(value, with_value, without_value)
144
- end
145
- else
146
- if with_value && !with_value.is_a?(Hash) && !without_unverified.has_key?(key)
147
- without_unverified[key] = value
148
- end
149
- end
144
+ def load_locale_directories(*paths)
145
+ paths.map do |path|
146
+ load_locale_directory(path)
150
147
  end
151
148
  end
152
149
 
153
- def clear_empty_strings!(hash)
154
- hash.delete_if do |key, value|
155
- if value == ""
156
- true
157
- elsif value.is_a?(Hash)
158
- clear_empty_strings!(value)
159
- value.empty?
160
- else
161
- false
162
- end
163
- end
164
- hash
150
+ def load_locale_directory(path)
151
+ @locale_file_class.load_directory(path)
165
152
  end
166
153
 
167
- class LocaleFileUpdates
168
- attr_reader :original_phraseapp_files, :resolved_files
154
+ def write_locale_file(path, locale_file)
155
+ File.write(path, locale_file.content)
156
+ end
169
157
 
170
- def initialize(original_phraseapp_files, resolved_files)
171
- @original_phraseapp_files = original_phraseapp_files
172
- @resolved_files = resolved_files
158
+ def write_locale_directory(path, locale_files)
159
+ locale_files.each do |locale_file|
160
+ full_path = File.join(path, locale_file.filename)
161
+ File.write(full_path, locale_file.content)
173
162
  end
163
+ STDERR.puts "Wrote #{locale_files.count} locale files to #{path}: #{locale_files.map(&:filename)}"
174
164
  end
175
165
  end
176
-
@@ -3,50 +3,91 @@ require "phraseapp_updater/locale_file/yaml_file"
3
3
 
4
4
  class PhraseAppUpdater
5
5
  class LocaleFile
6
- attr_reader :name, :content, :parsed_content
6
+ attr_reader :locale_name, :content, :parsed_content
7
7
 
8
- class BadFileTypeError < StandardError ; end
8
+ class BadFileTypeError < StandardError; end
9
9
 
10
- def self.from_hash(name, hash)
11
- raise RuntimeError.new("Must be implemented in a subclass.")
12
- end
10
+ class << self
11
+ def from_hash(locale_name, hash)
12
+ new(locale_name, hash)
13
+ end
13
14
 
14
- def self.class_for_file_format(type)
15
- case type.downcase
16
- when "json"
17
- JSONFile
18
- when "yml", "yaml"
19
- YAMLFile
20
- else
21
- raise BadFileTypeError.new("Invalid file type: #{type}")
15
+ def from_file_content(locale_name, content)
16
+ new(locale_name, load(content))
22
17
  end
23
- end
24
18
 
25
- # Expects a Ruby hash
26
- def initialize(name, content)
27
- @name = name
28
- @content = content
29
- @parsed_content = parse(@content)
30
- format_content!
19
+ def load_directory(directory)
20
+ Dir[File.join(directory, "*.#{extension}")].map do |filename|
21
+ load_file(filename)
22
+ end
23
+ end
24
+
25
+ def load_file(filename)
26
+ unless File.readable?(filename) && File.file?(filename)
27
+ raise RuntimeError.new("Couldn't read localization file at #{filename}")
28
+ end
29
+
30
+ locale_name = File.basename(filename).chomp(".#{extension}")
31
+ from_file_content(locale_name, File.read(filename))
32
+ end
33
+
34
+ private :new
35
+
36
+ def class_for_file_format(type)
37
+ case type.downcase
38
+ when "json"
39
+ JSONFile
40
+ when "yml", "yaml"
41
+ YAMLFile
42
+ else
43
+ raise BadFileTypeError.new("Invalid file type: #{type}")
44
+ end
45
+ end
46
+
47
+ def extension
48
+ raise RuntimeError.new("Abstract method")
49
+ end
50
+
51
+ def phraseapp_type
52
+ raise RuntimeError.new("Abstract method")
53
+ end
54
+
55
+ def load(_content)
56
+ raise RuntimeError.new("Abstract method")
57
+ end
58
+
59
+ def dump(_hash)
60
+ raise RuntimeError.new("Abstract method")
61
+ end
31
62
  end
32
63
 
33
64
  def to_s
34
- "#{name}, #{content[0,20]}..."
65
+ locale_name
35
66
  end
36
67
 
37
- def name_with_extension
38
- "#{name}.#{self.class::EXTENSION}"
68
+ def filename
69
+ "#{locale_name}.#{self.class.extension}"
39
70
  end
40
71
 
41
72
  private
42
73
 
43
- def parse(content)
44
- raise RuntimeError.new("Must be implemented in a subclass.")
74
+ # Expects a Ruby hash
75
+ def initialize(locale_name, parsed_content)
76
+ @locale_name = locale_name
77
+ @parsed_content = normalize_hash(parsed_content)
78
+ @content = self.class.dump(@parsed_content)
79
+ freeze
45
80
  end
46
81
 
47
- def format_content!
48
- raise RuntimeError.new("Must be implemented in a subclass.")
82
+ def normalize_hash(hash)
83
+ hash.keys.sort_by(&:to_s).each_with_object({}) do |key, normalized_hash|
84
+ val = hash[key]
85
+ next if val == '' || (val.is_a?(Hash) && val.empty?)
86
+
87
+ val = normalize_hash(val) if val.is_a?(Hash)
88
+ key = key.to_s if key.is_a?(Symbol)
89
+ normalized_hash[key] = val
90
+ end
49
91
  end
50
92
  end
51
93
  end
52
-