txgh 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (105) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +202 -0
  3. data/README.md +64 -0
  4. data/lib/ext/zipline/output_stream.rb +62 -0
  5. data/lib/txgh.rb +53 -0
  6. data/lib/txgh/app.rb +135 -0
  7. data/lib/txgh/category_support.rb +31 -0
  8. data/lib/txgh/config.rb +11 -0
  9. data/lib/txgh/config/config_pair.rb +36 -0
  10. data/lib/txgh/config/key_manager.rb +54 -0
  11. data/lib/txgh/config/provider_instance.rb +20 -0
  12. data/lib/txgh/config/provider_support.rb +26 -0
  13. data/lib/txgh/config/providers.rb +9 -0
  14. data/lib/txgh/config/providers/file_provider.rb +19 -0
  15. data/lib/txgh/config/providers/git_provider.rb +58 -0
  16. data/lib/txgh/config/providers/raw_provider.rb +19 -0
  17. data/lib/txgh/config/tx_config.rb +77 -0
  18. data/lib/txgh/config/tx_manager.rb +15 -0
  19. data/lib/txgh/diff_calculator.rb +90 -0
  20. data/lib/txgh/empty_resource_contents.rb +43 -0
  21. data/lib/txgh/errors.rb +9 -0
  22. data/lib/txgh/github_api.rb +83 -0
  23. data/lib/txgh/github_repo.rb +88 -0
  24. data/lib/txgh/github_request_auth.rb +28 -0
  25. data/lib/txgh/handlers.rb +12 -0
  26. data/lib/txgh/handlers/download_handler.rb +84 -0
  27. data/lib/txgh/handlers/github.rb +10 -0
  28. data/lib/txgh/handlers/github/delete_handler.rb +65 -0
  29. data/lib/txgh/handlers/github/handler.rb +20 -0
  30. data/lib/txgh/handlers/github/push_handler.rb +108 -0
  31. data/lib/txgh/handlers/github/request_handler.rb +106 -0
  32. data/lib/txgh/handlers/response.rb +17 -0
  33. data/lib/txgh/handlers/stream_response.rb +39 -0
  34. data/lib/txgh/handlers/tgz_stream_response.rb +41 -0
  35. data/lib/txgh/handlers/transifex.rb +8 -0
  36. data/lib/txgh/handlers/transifex/hook_handler.rb +77 -0
  37. data/lib/txgh/handlers/transifex/request_handler.rb +78 -0
  38. data/lib/txgh/handlers/triggers.rb +9 -0
  39. data/lib/txgh/handlers/triggers/handler.rb +66 -0
  40. data/lib/txgh/handlers/triggers/pull_handler.rb +29 -0
  41. data/lib/txgh/handlers/triggers/push_handler.rb +21 -0
  42. data/lib/txgh/handlers/zip_stream_response.rb +21 -0
  43. data/lib/txgh/merge_calculator.rb +74 -0
  44. data/lib/txgh/parse_config.rb +24 -0
  45. data/lib/txgh/resource_committer.rb +39 -0
  46. data/lib/txgh/resource_contents.rb +118 -0
  47. data/lib/txgh/resource_downloader.rb +141 -0
  48. data/lib/txgh/resource_updater.rb +104 -0
  49. data/lib/txgh/response_helpers.rb +30 -0
  50. data/lib/txgh/transifex_api.rb +165 -0
  51. data/lib/txgh/transifex_project.rb +37 -0
  52. data/lib/txgh/transifex_request_auth.rb +53 -0
  53. data/lib/txgh/tx_branch_resource.rb +59 -0
  54. data/lib/txgh/tx_logger.rb +12 -0
  55. data/lib/txgh/tx_resource.rb +66 -0
  56. data/lib/txgh/utils.rb +44 -0
  57. data/lib/txgh/version.rb +3 -0
  58. data/spec/app_spec.rb +346 -0
  59. data/spec/category_support_spec.rb +43 -0
  60. data/spec/config/config_pair_spec.rb +47 -0
  61. data/spec/config/key_manager_spec.rb +48 -0
  62. data/spec/config/provider_instance_spec.rb +30 -0
  63. data/spec/config/provider_support_spec.rb +55 -0
  64. data/spec/config/tx_config_spec.rb +49 -0
  65. data/spec/config/tx_manager_spec.rb +57 -0
  66. data/spec/diff_calculator_spec.rb +90 -0
  67. data/spec/github_api_spec.rb +148 -0
  68. data/spec/github_repo_spec.rb +178 -0
  69. data/spec/github_request_auth_spec.rb +39 -0
  70. data/spec/handlers/download_handler_spec.rb +81 -0
  71. data/spec/handlers/github/delete_handler_spec.rb +71 -0
  72. data/spec/handlers/github/push_handler_spec.rb +76 -0
  73. data/spec/handlers/tgz_stream_response_spec.rb +59 -0
  74. data/spec/handlers/transifex/hook_handler_spec.rb +115 -0
  75. data/spec/handlers/zip_stream_response_spec.rb +58 -0
  76. data/spec/helpers/github_payload_builder.rb +141 -0
  77. data/spec/helpers/integration_setup.rb +47 -0
  78. data/spec/helpers/nil_logger.rb +10 -0
  79. data/spec/helpers/standard_txgh_setup.rb +92 -0
  80. data/spec/helpers/test_provider.rb +12 -0
  81. data/spec/integration/cassettes/github_l10n_hook_endpoint.yml +536 -0
  82. data/spec/integration/cassettes/pull.yml +47 -0
  83. data/spec/integration/cassettes/push.yml +544 -0
  84. data/spec/integration/cassettes/transifex_hook_endpoint.yml +560 -0
  85. data/spec/integration/config/tx.config +10 -0
  86. data/spec/integration/hooks_spec.rb +158 -0
  87. data/spec/integration/payloads/github_postbody.json +161 -0
  88. data/spec/integration/payloads/github_postbody_l10n.json +136 -0
  89. data/spec/integration/payloads/github_postbody_release.json +136 -0
  90. data/spec/integration/triggers_spec.rb +45 -0
  91. data/spec/merge_calculator_spec.rb +112 -0
  92. data/spec/parse_config_spec.rb +52 -0
  93. data/spec/resource_committer_spec.rb +42 -0
  94. data/spec/resource_contents_spec.rb +212 -0
  95. data/spec/resource_downloader_spec.rb +205 -0
  96. data/spec/resource_updater_spec.rb +147 -0
  97. data/spec/spec_helper.rb +32 -0
  98. data/spec/transifex_api_spec.rb +345 -0
  99. data/spec/transifex_project_spec.rb +45 -0
  100. data/spec/transifex_request_auth_spec.rb +39 -0
  101. data/spec/tx_branch_resource_spec.rb +99 -0
  102. data/spec/tx_resource_spec.rb +47 -0
  103. data/spec/utils_spec.rb +58 -0
  104. data/txgh.gemspec +29 -0
  105. metadata +296 -0
@@ -0,0 +1,104 @@
1
+ require 'logger'
2
+
3
+ module Txgh
4
+ class ResourceUpdater
5
+ include Txgh::CategorySupport
6
+
7
+ attr_reader :project, :repo, :logger
8
+
9
+ def initialize(project, repo, logger = nil)
10
+ @project = project
11
+ @repo = repo
12
+ @logger = logger || Logger.new(STDOUT)
13
+ end
14
+
15
+ # For each modified resource, get its content and update the content
16
+ # in Transifex.
17
+ def update_resource(tx_resource, commit_sha, categories = {})
18
+ logger.info('process updated resource')
19
+ tree_sha = repo.api.get_commit(repo.name, commit_sha)['commit']['tree']['sha']
20
+ tree = repo.api.tree(repo.name, tree_sha)
21
+
22
+ tree['tree'].each do |file|
23
+ logger.info("process each tree entry: #{file['path']}")
24
+
25
+ if tx_resource.source_file == file['path']
26
+ if repo.upload_diffs?
27
+ upload_diff(tx_resource, file, categories)
28
+ else
29
+ upload_whole(tx_resource, file, categories)
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def upload_whole(tx_resource, file, categories)
38
+ content = contents_of(file['sha'])
39
+
40
+ if repo.process_all_branches?
41
+ upload_by_branch(tx_resource, content, categories)
42
+ else
43
+ upload(tx_resource, content)
44
+ end
45
+ end
46
+
47
+ def upload_diff(tx_resource, file, categories)
48
+ if content = diff_content(tx_resource, file)
49
+ upload_by_branch(tx_resource, content, categories)
50
+ end
51
+ end
52
+
53
+ def diff_content(tx_resource, file)
54
+ diff = head_content(tx_resource, file).diff(
55
+ diff_point_content(tx_resource, file)
56
+ )
57
+
58
+ diff.to_s unless diff.empty?
59
+ end
60
+
61
+ def head_content(tx_resource, file)
62
+ ResourceContents.from_string(tx_resource, contents_of(file['sha']))
63
+ end
64
+
65
+ def diff_point_content(tx_resource, file)
66
+ raw_content = repo.api.download(repo.name, file['path'], repo.diff_point)
67
+ ResourceContents.from_string(tx_resource, raw_content)
68
+ end
69
+
70
+ def upload(tx_resource, content)
71
+ project.api.create_or_update(tx_resource, content)
72
+ end
73
+
74
+ def upload_by_branch(tx_resource, content, additional_categories)
75
+ resource_exists = project.api.resource_exists?(tx_resource)
76
+ categories = resource_exists ? categories_for(tx_resource) : {}
77
+ categories.merge!(additional_categories)
78
+ categories['branch'] ||= tx_resource.branch
79
+ categories = serialize_categories(categories)
80
+
81
+ if resource_exists
82
+ project.api.update_details(tx_resource, categories: categories)
83
+ project.api.update_content(tx_resource, content)
84
+ else
85
+ project.api.create(tx_resource, content, categories)
86
+ end
87
+ end
88
+
89
+ def categories_for(tx_resource)
90
+ resource = project.api.get_resource(*tx_resource.slugs)
91
+ deserialize_categories(Array(resource['categories']))
92
+ end
93
+
94
+ def contents_of(sha)
95
+ blob = repo.api.blob(repo.name, sha)
96
+
97
+ if blob['encoding'] == 'utf-8'
98
+ blob['content']
99
+ else
100
+ Base64.decode64(blob['content'])
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,30 @@
1
+ module Txgh
2
+ module ResponseHelpers
3
+ private
4
+
5
+ def respond_with(status, body, e = nil)
6
+ Txgh::Handlers::Response.new(status, body, e)
7
+ end
8
+
9
+ def respond_with_success(status, body)
10
+ respond_with(status, data(body))
11
+ end
12
+
13
+ def respond_with_error(status, message, e = nil)
14
+ respond_with(status, error(message), e)
15
+ end
16
+
17
+ def error(message)
18
+ [{ error: message }]
19
+ end
20
+
21
+ def data(body)
22
+ { data: body }
23
+ end
24
+
25
+ # includes these methods in the singleton class as well
26
+ def self.included(base)
27
+ base.extend(self)
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,165 @@
1
+ require 'faraday'
2
+ require 'faraday_middleware'
3
+ require 'json'
4
+ require 'set'
5
+
6
+ module Txgh
7
+ class TransifexApi
8
+ API_ROOT = '/api/2'
9
+
10
+ class << self
11
+ def create_from_credentials(username, password)
12
+ connection = Faraday.new(url: 'https://www.transifex.com') do |faraday|
13
+ faraday.request(:multipart)
14
+ faraday.request(:json)
15
+ faraday.request(:url_encoded)
16
+
17
+ faraday.response(:logger)
18
+ faraday.use(FaradayMiddleware::FollowRedirects)
19
+ faraday.adapter(Faraday.default_adapter)
20
+ end
21
+
22
+ connection.basic_auth(username, password)
23
+ connection.headers.update(Accept: 'application/json')
24
+ create_from_connection(connection)
25
+ end
26
+
27
+ def create_from_connection(connection)
28
+ new(connection)
29
+ end
30
+ end
31
+
32
+ attr_reader :connection
33
+
34
+ def initialize(connection)
35
+ @connection = connection
36
+ end
37
+
38
+ def create_or_update(tx_resource, content, categories = [])
39
+ if resource_exists?(tx_resource)
40
+ resource = get_resource(*tx_resource.slugs)
41
+ new_categories = Set.new(resource['categories'])
42
+ new_categories.merge(categories)
43
+
44
+ # update details first so new content is always tagged
45
+ update_details(tx_resource, categories: new_categories.to_a)
46
+ update_content(tx_resource, content)
47
+ else
48
+ create(tx_resource, content, categories)
49
+ end
50
+ end
51
+
52
+ def create(tx_resource, content, categories = [])
53
+ payload = {
54
+ slug: tx_resource.resource_slug,
55
+ name: tx_resource.source_file,
56
+ i18n_type: tx_resource.type,
57
+ categories: CategorySupport.join_categories(categories.uniq),
58
+ content: get_content_io(tx_resource, content)
59
+ }
60
+
61
+ url = "#{API_ROOT}/project/#{tx_resource.project_slug}/resources/"
62
+ response = connection.post(url, payload)
63
+ raise_error!(response)
64
+ end
65
+
66
+ def delete(tx_resource)
67
+ url = "#{API_ROOT}/project/#{tx_resource.project_slug}/resource/#{tx_resource.resource_slug}/"
68
+ connection.delete(url)
69
+ end
70
+
71
+ def update_content(tx_resource, content)
72
+ content_io = get_content_io(tx_resource, content)
73
+ payload = { content: content_io }
74
+ url = "#{API_ROOT}/project/#{tx_resource.project_slug}/resource/#{tx_resource.resource_slug}/content/"
75
+ response = connection.put(url, payload)
76
+ raise_error!(response)
77
+ end
78
+
79
+ def update_details(tx_resource, details = {})
80
+ url = "#{API_ROOT}/project/#{tx_resource.project_slug}/resource/#{tx_resource.resource_slug}/"
81
+ response = connection.put(url, details)
82
+ raise_error!(response)
83
+ end
84
+
85
+ def resource_exists?(tx_resource)
86
+ project = tx_resource.project_slug
87
+ slug = tx_resource.resource_slug
88
+ response = connection.get("#{API_ROOT}/project/#{project}/resource/#{slug}/")
89
+ response.status == 200
90
+ end
91
+
92
+ def download(tx_resource, lang)
93
+ project_slug = tx_resource.project_slug
94
+ resource_slug = tx_resource.resource_slug
95
+ response = connection.get(
96
+ "#{API_ROOT}/project/#{project_slug}/resource/#{resource_slug}/translation/#{lang}/"
97
+ )
98
+
99
+ raise_error!(response)
100
+
101
+ json_data = JSON.parse(response.body)
102
+ json_data['content']
103
+ end
104
+
105
+ def get_resource(project_slug, resource_slug)
106
+ url = "#{API_ROOT}/project/#{project_slug}/resource/#{resource_slug}/"
107
+ response = connection.get(url)
108
+ raise_error!(response)
109
+ JSON.parse(response.body)
110
+ end
111
+
112
+ def get_resources(project_slug)
113
+ url = "#{API_ROOT}/project/#{project_slug}/resources/"
114
+ response = connection.get(url)
115
+ raise_error!(response)
116
+ JSON.parse(response.body)
117
+ end
118
+
119
+ def get_languages(project_slug)
120
+ url = "#{API_ROOT}/project/#{project_slug}/languages/"
121
+ response = connection.get(url)
122
+ raise_error!(response)
123
+ JSON.parse(response.body)
124
+ end
125
+
126
+ def get_project(project_slug)
127
+ url = "#{API_ROOT}/project/#{project_slug}/"
128
+ response = connection.get(url)
129
+ raise_error!(response)
130
+ JSON.parse(response.body)
131
+ end
132
+
133
+ def get_formats
134
+ url = "#{API_ROOT}/formats/"
135
+ response = connection.get(url)
136
+ raise_error!(response)
137
+ JSON.parse(response.body)
138
+ end
139
+
140
+ private
141
+
142
+ def get_content_io(tx_resource, content)
143
+ content_io = StringIO::new(content)
144
+ content_io.set_encoding(Encoding::UTF_8.name)
145
+ Faraday::UploadIO.new(
146
+ content_io, 'application/octet-stream', tx_resource.source_file
147
+ )
148
+ end
149
+
150
+ def raise_error!(response)
151
+ case response.status
152
+ when 401
153
+ raise TransifexUnauthorizedError
154
+ when 404
155
+ raise TransifexNotFoundError
156
+ else
157
+ if (response.status / 100) != 2
158
+ raise TransifexApiError,
159
+ "Failed Transifex API call - returned status code: #{response.status}, body: #{response.body}"
160
+ end
161
+ end
162
+ end
163
+ end
164
+ end
165
+
@@ -0,0 +1,37 @@
1
+ module Txgh
2
+ class TransifexProject
3
+ attr_reader :config, :api
4
+
5
+ def initialize(config, api)
6
+ @config = config
7
+ @api = api
8
+ end
9
+
10
+ def name
11
+ config['name']
12
+ end
13
+
14
+ def webhook_secret
15
+ config['webhook_secret']
16
+ end
17
+
18
+ def protected_branches
19
+ @protected_branches ||=
20
+ (config['protected_branches'] || '').split(',').map do |branch|
21
+ Utils.absolute_branch(branch.strip)
22
+ end
23
+ end
24
+
25
+ def webhook_protected?
26
+ !(webhook_secret || '').empty?
27
+ end
28
+
29
+ def auto_delete_resources?
30
+ (config['auto_delete_resources'] || '').downcase == 'true'
31
+ end
32
+
33
+ def tx_config_uri
34
+ config['tx_config']
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,53 @@
1
+ require 'openssl'
2
+
3
+ module Txgh
4
+ class TransifexRequestAuth
5
+ HMAC_DIGEST = OpenSSL::Digest.new('sha1')
6
+ RACK_HEADER = 'HTTP_X_TX_SIGNATURE'
7
+ TRANSIFEX_HEADER = 'X-TX-Signature'
8
+
9
+ class << self
10
+ def authentic_request?(request, secret)
11
+ request.body.rewind
12
+ expected_signature = header_value(request.body.read, secret)
13
+ actual_signature = request.env[RACK_HEADER]
14
+ actual_signature == expected_signature
15
+ end
16
+
17
+ def header_value(content, secret)
18
+ digest(transform(content), secret)
19
+ end
20
+
21
+ private
22
+
23
+ # In order to generate a correct HMAC hash, the request body must be
24
+ # parsed and made to look like a python map. If you're thinking that's
25
+ # weird, you're correct, but it's apparently expected behavior.
26
+ def transform(content)
27
+ params = URI.decode_www_form(content)
28
+
29
+ params = params.map do |key, val|
30
+ key = "'#{key}'"
31
+ val = interpret_val(val)
32
+ "#{key}: #{val}"
33
+ end
34
+
35
+ "{#{params.join(', ')}}"
36
+ end
37
+
38
+ def interpret_val(val)
39
+ if val =~ /\A[\d]+\z/
40
+ val
41
+ else
42
+ "u'#{val}'"
43
+ end
44
+ end
45
+
46
+ def digest(content, secret)
47
+ Base64.encode64(
48
+ OpenSSL::HMAC.digest(HMAC_DIGEST, secret, content)
49
+ ).strip
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,59 @@
1
+ require 'forwardable'
2
+
3
+ module Txgh
4
+ class TxBranchResource
5
+ extend Forwardable
6
+
7
+ def_delegators :@resource, *[
8
+ :project_slug, :type, :source_lang, :source_file, :L10N_resource_slug,
9
+ :translation_file, :lang_map, :translation_path, :original_resource_slug,
10
+ :to_h, :to_api_h
11
+ ]
12
+
13
+ attr_reader :resource, :branch
14
+
15
+ class << self
16
+ def find(tx_config, resource_slug, branch)
17
+ resource_slug = deslugify(resource_slug, branch)
18
+ resource = tx_config.resource(resource_slug)
19
+ new(resource, branch) if resource
20
+ end
21
+
22
+ def deslugify(resource_slug, branch)
23
+ suffix = "-#{Utils.slugify(branch)}"
24
+
25
+ if resource_slug.end_with?(suffix)
26
+ resource_slug.chomp(suffix)
27
+ else
28
+ resource_slug
29
+ end
30
+ end
31
+ end
32
+
33
+ def initialize(resource, branch)
34
+ @resource = resource
35
+ @branch = branch
36
+ end
37
+
38
+ def resource_slug
39
+ "#{resource.resource_slug}-#{slugified_branch}"
40
+ end
41
+
42
+ def slugs
43
+ [project_slug, resource_slug]
44
+ end
45
+
46
+ def to_h
47
+ resource.to_h.merge(
48
+ project_slug: project_slug,
49
+ resource_slug: resource_slug
50
+ )
51
+ end
52
+
53
+ private
54
+
55
+ def slugified_branch
56
+ Utils.slugify(branch)
57
+ end
58
+ end
59
+ end