txgh 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (105) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +202 -0
  3. data/README.md +64 -0
  4. data/lib/ext/zipline/output_stream.rb +62 -0
  5. data/lib/txgh.rb +53 -0
  6. data/lib/txgh/app.rb +135 -0
  7. data/lib/txgh/category_support.rb +31 -0
  8. data/lib/txgh/config.rb +11 -0
  9. data/lib/txgh/config/config_pair.rb +36 -0
  10. data/lib/txgh/config/key_manager.rb +54 -0
  11. data/lib/txgh/config/provider_instance.rb +20 -0
  12. data/lib/txgh/config/provider_support.rb +26 -0
  13. data/lib/txgh/config/providers.rb +9 -0
  14. data/lib/txgh/config/providers/file_provider.rb +19 -0
  15. data/lib/txgh/config/providers/git_provider.rb +58 -0
  16. data/lib/txgh/config/providers/raw_provider.rb +19 -0
  17. data/lib/txgh/config/tx_config.rb +77 -0
  18. data/lib/txgh/config/tx_manager.rb +15 -0
  19. data/lib/txgh/diff_calculator.rb +90 -0
  20. data/lib/txgh/empty_resource_contents.rb +43 -0
  21. data/lib/txgh/errors.rb +9 -0
  22. data/lib/txgh/github_api.rb +83 -0
  23. data/lib/txgh/github_repo.rb +88 -0
  24. data/lib/txgh/github_request_auth.rb +28 -0
  25. data/lib/txgh/handlers.rb +12 -0
  26. data/lib/txgh/handlers/download_handler.rb +84 -0
  27. data/lib/txgh/handlers/github.rb +10 -0
  28. data/lib/txgh/handlers/github/delete_handler.rb +65 -0
  29. data/lib/txgh/handlers/github/handler.rb +20 -0
  30. data/lib/txgh/handlers/github/push_handler.rb +108 -0
  31. data/lib/txgh/handlers/github/request_handler.rb +106 -0
  32. data/lib/txgh/handlers/response.rb +17 -0
  33. data/lib/txgh/handlers/stream_response.rb +39 -0
  34. data/lib/txgh/handlers/tgz_stream_response.rb +41 -0
  35. data/lib/txgh/handlers/transifex.rb +8 -0
  36. data/lib/txgh/handlers/transifex/hook_handler.rb +77 -0
  37. data/lib/txgh/handlers/transifex/request_handler.rb +78 -0
  38. data/lib/txgh/handlers/triggers.rb +9 -0
  39. data/lib/txgh/handlers/triggers/handler.rb +66 -0
  40. data/lib/txgh/handlers/triggers/pull_handler.rb +29 -0
  41. data/lib/txgh/handlers/triggers/push_handler.rb +21 -0
  42. data/lib/txgh/handlers/zip_stream_response.rb +21 -0
  43. data/lib/txgh/merge_calculator.rb +74 -0
  44. data/lib/txgh/parse_config.rb +24 -0
  45. data/lib/txgh/resource_committer.rb +39 -0
  46. data/lib/txgh/resource_contents.rb +118 -0
  47. data/lib/txgh/resource_downloader.rb +141 -0
  48. data/lib/txgh/resource_updater.rb +104 -0
  49. data/lib/txgh/response_helpers.rb +30 -0
  50. data/lib/txgh/transifex_api.rb +165 -0
  51. data/lib/txgh/transifex_project.rb +37 -0
  52. data/lib/txgh/transifex_request_auth.rb +53 -0
  53. data/lib/txgh/tx_branch_resource.rb +59 -0
  54. data/lib/txgh/tx_logger.rb +12 -0
  55. data/lib/txgh/tx_resource.rb +66 -0
  56. data/lib/txgh/utils.rb +44 -0
  57. data/lib/txgh/version.rb +3 -0
  58. data/spec/app_spec.rb +346 -0
  59. data/spec/category_support_spec.rb +43 -0
  60. data/spec/config/config_pair_spec.rb +47 -0
  61. data/spec/config/key_manager_spec.rb +48 -0
  62. data/spec/config/provider_instance_spec.rb +30 -0
  63. data/spec/config/provider_support_spec.rb +55 -0
  64. data/spec/config/tx_config_spec.rb +49 -0
  65. data/spec/config/tx_manager_spec.rb +57 -0
  66. data/spec/diff_calculator_spec.rb +90 -0
  67. data/spec/github_api_spec.rb +148 -0
  68. data/spec/github_repo_spec.rb +178 -0
  69. data/spec/github_request_auth_spec.rb +39 -0
  70. data/spec/handlers/download_handler_spec.rb +81 -0
  71. data/spec/handlers/github/delete_handler_spec.rb +71 -0
  72. data/spec/handlers/github/push_handler_spec.rb +76 -0
  73. data/spec/handlers/tgz_stream_response_spec.rb +59 -0
  74. data/spec/handlers/transifex/hook_handler_spec.rb +115 -0
  75. data/spec/handlers/zip_stream_response_spec.rb +58 -0
  76. data/spec/helpers/github_payload_builder.rb +141 -0
  77. data/spec/helpers/integration_setup.rb +47 -0
  78. data/spec/helpers/nil_logger.rb +10 -0
  79. data/spec/helpers/standard_txgh_setup.rb +92 -0
  80. data/spec/helpers/test_provider.rb +12 -0
  81. data/spec/integration/cassettes/github_l10n_hook_endpoint.yml +536 -0
  82. data/spec/integration/cassettes/pull.yml +47 -0
  83. data/spec/integration/cassettes/push.yml +544 -0
  84. data/spec/integration/cassettes/transifex_hook_endpoint.yml +560 -0
  85. data/spec/integration/config/tx.config +10 -0
  86. data/spec/integration/hooks_spec.rb +158 -0
  87. data/spec/integration/payloads/github_postbody.json +161 -0
  88. data/spec/integration/payloads/github_postbody_l10n.json +136 -0
  89. data/spec/integration/payloads/github_postbody_release.json +136 -0
  90. data/spec/integration/triggers_spec.rb +45 -0
  91. data/spec/merge_calculator_spec.rb +112 -0
  92. data/spec/parse_config_spec.rb +52 -0
  93. data/spec/resource_committer_spec.rb +42 -0
  94. data/spec/resource_contents_spec.rb +212 -0
  95. data/spec/resource_downloader_spec.rb +205 -0
  96. data/spec/resource_updater_spec.rb +147 -0
  97. data/spec/spec_helper.rb +32 -0
  98. data/spec/transifex_api_spec.rb +345 -0
  99. data/spec/transifex_project_spec.rb +45 -0
  100. data/spec/transifex_request_auth_spec.rb +39 -0
  101. data/spec/tx_branch_resource_spec.rb +99 -0
  102. data/spec/tx_resource_spec.rb +47 -0
  103. data/spec/utils_spec.rb +58 -0
  104. data/txgh.gemspec +29 -0
  105. metadata +296 -0
@@ -0,0 +1,31 @@
1
+ module Txgh
2
+ module CategorySupport
3
+ def deserialize_categories(categories_arr)
4
+ categories_arr.each_with_object({}) do |category_str, ret|
5
+ category_str.split(' ').each do |category|
6
+ if idx = category.index(':')
7
+ ret[category[0...idx]] = category[(idx + 1)..-1]
8
+ end
9
+ end
10
+ end
11
+ end
12
+
13
+ def serialize_categories(categories_hash)
14
+ categories_hash.map do |key, value|
15
+ "#{key}:#{escape_category(value)}"
16
+ end
17
+ end
18
+
19
+ def escape_category(str)
20
+ str.gsub(' ', '_')
21
+ end
22
+
23
+ def join_categories(arr)
24
+ arr.join(' ')
25
+ end
26
+ end
27
+
28
+ # add all the methods as class methods (they're also available as instance
29
+ # methods for anyone who includes this module)
30
+ CategorySupport.extend(CategorySupport)
31
+ end
@@ -0,0 +1,11 @@
1
+ module Txgh
2
+ module Config
3
+ autoload :ConfigPair, 'txgh/config/config_pair'
4
+ autoload :KeyManager, 'txgh/config/key_manager'
5
+ autoload :ProviderInstance, 'txgh/config/provider_instance'
6
+ autoload :Providers, 'txgh/config/providers'
7
+ autoload :ProviderSupport, 'txgh/config/provider_support'
8
+ autoload :TxConfig, 'txgh/config/tx_config'
9
+ autoload :TxManager, 'txgh/config/tx_manager'
10
+ end
11
+ end
@@ -0,0 +1,36 @@
1
+ module Txgh
2
+ module Config
3
+ class ConfigPair
4
+ attr_reader :project_config, :repo_config
5
+
6
+ def initialize(project_config, repo_config)
7
+ @project_config = project_config
8
+ @repo_config = repo_config
9
+ end
10
+
11
+ def github_repo
12
+ @github_repo ||= Txgh::GithubRepo.new(
13
+ repo_config, github_api
14
+ )
15
+ end
16
+
17
+ def transifex_project
18
+ @transifex_project ||= Txgh::TransifexProject.new(
19
+ project_config, transifex_api
20
+ )
21
+ end
22
+
23
+ def transifex_api
24
+ @transifex_api ||= Txgh::TransifexApi.create_from_credentials(
25
+ project_config['api_username'], project_config['api_password']
26
+ )
27
+ end
28
+
29
+ def github_api
30
+ @github_api ||= Txgh::GithubApi.create_from_credentials(
31
+ repo_config['api_username'], repo_config['api_token']
32
+ )
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,54 @@
1
+ require 'erb'
2
+ require 'etc'
3
+ require 'yaml'
4
+
5
+ module Txgh
6
+ module Config
7
+ class KeyManager
8
+ class << self
9
+ include ProviderSupport
10
+
11
+ def config_from_project(project_name)
12
+ project_config = project_config_for(project_name)
13
+ repo_config = repo_config_for(project_config['push_translations_to'])
14
+ ConfigPair.new(project_config, repo_config)
15
+ end
16
+
17
+ def config_from_repo(repo_name)
18
+ repo_config = repo_config_for(repo_name)
19
+ project_config = project_config_for(repo_config['push_source_to'])
20
+ ConfigPair.new(project_config, repo_config)
21
+ end
22
+
23
+ def config_from(project_name, repo_name)
24
+ project_config = project_config_for(project_name)
25
+ repo_config = repo_config_for(repo_name)
26
+ ConfigPair.new(project_config, repo_config)
27
+ end
28
+
29
+ private
30
+
31
+ def raw_config
32
+ ENV['TXGH_CONFIG']
33
+ end
34
+
35
+ def base_config
36
+ scheme, payload = split_uri(raw_config)
37
+ provider_for(scheme).load(payload)
38
+ end
39
+
40
+ def project_config_for(project_name)
41
+ if config = base_config['transifex']['projects'][project_name]
42
+ config.merge('name' => project_name)
43
+ end
44
+ end
45
+
46
+ def repo_config_for(repo_name)
47
+ if config = base_config['github']['repos'][repo_name]
48
+ config.merge('name' => repo_name)
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,20 @@
1
+ module Txgh
2
+ module Config
3
+ class ProviderInstance
4
+ attr_reader :provider, :parser
5
+
6
+ def initialize(provider, parser)
7
+ @provider = provider
8
+ @parser = parser
9
+ end
10
+
11
+ def supports?(*args)
12
+ provider.supports?(*args)
13
+ end
14
+
15
+ def load(payload, options = {})
16
+ provider.load(payload, parser, options)
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,26 @@
1
+ module Txgh
2
+ module Config
3
+ module ProviderSupport
4
+ def register_provider(provider, parser)
5
+ providers << ProviderInstance.new(provider, parser)
6
+ end
7
+
8
+ def providers
9
+ @providers ||= []
10
+ end
11
+
12
+ def provider_for(scheme)
13
+ providers.find { |provider| provider.supports?(scheme) }
14
+ end
15
+
16
+ def split_uri(uri)
17
+ if uri =~ /\A[\w]+:\/\//
18
+ idx = uri.index('://')
19
+ [uri[0...idx], uri[(idx + 3)..-1]]
20
+ else
21
+ [nil, uri]
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,9 @@
1
+ module Txgh
2
+ module Config
3
+ module Providers
4
+ autoload :FileProvider, 'txgh/config/providers/file_provider'
5
+ autoload :GitProvider, 'txgh/config/providers/git_provider'
6
+ autoload :RawProvider, 'txgh/config/providers/raw_provider'
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,19 @@
1
+ module Txgh
2
+ module Config
3
+ module Providers
4
+ class FileProvider
5
+ SCHEME = 'file'
6
+
7
+ class << self
8
+ def supports?(scheme)
9
+ scheme == SCHEME
10
+ end
11
+
12
+ def load(payload, parser, options = {})
13
+ parser.load_file(payload)
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,58 @@
1
+ module Txgh
2
+ module Config
3
+ module Providers
4
+ class GitProvider
5
+ SCHEME = 'git'
6
+
7
+ class << self
8
+ def supports?(scheme)
9
+ scheme == SCHEME
10
+ end
11
+
12
+ def load(payload, parser, options = {})
13
+ new(payload, parser, options).config
14
+ end
15
+ end
16
+
17
+ attr_reader :payload, :parser
18
+
19
+ def initialize(payload, parser, options = {})
20
+ @payload = payload
21
+ @parser = parser
22
+ @ref = options[:ref]
23
+ @github_repo = options[:github_repo]
24
+ end
25
+
26
+ def config
27
+ parser.load(download)
28
+ end
29
+
30
+ private
31
+
32
+ def download
33
+ github_repo.api.download(github_repo.name, payload, ref)
34
+ rescue Octokit::NotFound
35
+ raise ConfigNotFoundError, "Config file #{payload} not found in #{ref}"
36
+ end
37
+
38
+ def ref
39
+ unless @ref
40
+ raise TxghError,
41
+ "TX_CONFIG specified a file from git but did not provide a ref."
42
+ end
43
+
44
+ @ref
45
+ end
46
+
47
+ def github_repo
48
+ unless @github_repo
49
+ raise TxghError,
50
+ "TX_CONFIG specified a file from git but did not provide a repo."
51
+ end
52
+
53
+ @github_repo
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,19 @@
1
+ module Txgh
2
+ module Config
3
+ module Providers
4
+ class RawProvider
5
+ SCHEME = 'raw'
6
+
7
+ class << self
8
+ def supports?(scheme)
9
+ scheme == SCHEME
10
+ end
11
+
12
+ def load(payload, parser, options = {})
13
+ parser.load(payload)
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,77 @@
1
+ module Txgh
2
+ module Config
3
+ class TxConfig
4
+ class << self
5
+ def load_file(path)
6
+ config = Txgh::ParseConfig.load_file(path)
7
+ parse(config)
8
+ end
9
+
10
+ def load(contents)
11
+ config = Txgh::ParseConfig.load(contents)
12
+ parse(config)
13
+ end
14
+
15
+ private
16
+
17
+ def parse(config)
18
+ resources = []
19
+ lang_map = {}
20
+
21
+ config.get_groups.each do |group|
22
+ if group == 'main'
23
+ main = config[group]
24
+
25
+ if main['lang_map']
26
+ lang_map = parse_lang_map(main['lang_map'])
27
+ end
28
+ else
29
+ resources.push(
30
+ parse_resource(group, config[group])
31
+ )
32
+ end
33
+ end
34
+
35
+ new(resources, lang_map)
36
+ end
37
+
38
+ def parse_lang_map(lang_map)
39
+ lang_map.split(',').each_with_object({}) do |m, result|
40
+ key_value = m.split(':', 2)
41
+ result[key_value[0].strip] = key_value[1].strip
42
+ end
43
+ end
44
+
45
+ def parse_resource(name, resource)
46
+ id = name.split('.', 2)
47
+ TxResource.new(
48
+ id[0].strip, id[1].strip, resource['type'],
49
+ resource['source_lang'], resource['source_file'],
50
+ resource['lang_map'], resource['file_filter']
51
+ )
52
+ end
53
+ end
54
+
55
+ attr_reader :resources, :lang_map
56
+
57
+ def initialize(resources, lang_map)
58
+ @resources = resources
59
+ @lang_map = lang_map
60
+ end
61
+
62
+ def resource(slug, branch = nil)
63
+ if branch
64
+ TxBranchResource.find(self, slug, branch)
65
+ else
66
+ resources.find do |resource|
67
+ resource.resource_slug == slug
68
+ end
69
+ end
70
+ end
71
+
72
+ def to_h
73
+ { resources: resources.map(&:to_h), lang_map: lang_map }
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,15 @@
1
+ module Txgh
2
+ module Config
3
+ class TxManager
4
+ class << self
5
+ include ProviderSupport
6
+
7
+ def tx_config(transifex_project, github_repo, ref = nil)
8
+ options = { github_repo: github_repo, ref: ref }
9
+ scheme, payload = split_uri(transifex_project.tx_config_uri)
10
+ provider_for(scheme).load(payload, options)
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,90 @@
1
+ module Txgh
2
+ class DiffCalculator
3
+ class << self
4
+ def compare(head_phrases, diff_point_phrases)
5
+ new(head_phrases, diff_point_phrases).compare
6
+ end
7
+ end
8
+
9
+ INCLUDED_STATES = [:added, :modified, :removed]
10
+
11
+ attr_reader :head_phrases, :diff_point_phrases
12
+
13
+ def initialize(head_phrases, diff_point_phrases)
14
+ @head_phrases = head_phrases
15
+ @diff_point_phrases = diff_point_phrases
16
+ end
17
+
18
+ def compare
19
+ join_diffs(compare_head, compare_diff_point)
20
+ end
21
+
22
+ private
23
+
24
+ def join_diffs(diff1, diff2)
25
+ INCLUDED_STATES.each_with_object({}) do |state, ret|
26
+ ret[state] = diff1[state] + diff2[state]
27
+ end
28
+ end
29
+
30
+ def compare_head
31
+ compare_enum(head_to_diff_point)
32
+ end
33
+
34
+ def compare_diff_point
35
+ compare_enum(diff_point_to_head)
36
+ end
37
+
38
+ def compare_enum(enum)
39
+ enum.each_with_object(new_hash_of_arrays) do |(phrase, state), diff|
40
+ diff[state] << phrase
41
+ end
42
+ end
43
+
44
+ # identifies phrases in head that:
45
+ # are not in diff point ('added')
46
+ # have the same key but different strings diff point ('modified')
47
+ # are identical to a phrase in diff point ('unmodified'), but
48
+ # does not yield them
49
+ def head_to_diff_point
50
+ return to_enum(__method__) unless block_given?
51
+
52
+ # iterate over all head phrases that have keys
53
+ head_phrases.each do |head_phrase|
54
+ idx = diff_point_phrases.find_index do |diff_point_phrase|
55
+ diff_point_phrase['key'] == head_phrase['key']
56
+ end
57
+
58
+ state = compare_at(idx, head_phrase)
59
+ yield(head_phrase, state) if INCLUDED_STATES.include?(state)
60
+ end
61
+ end
62
+
63
+ def compare_at(idx, head_phrase)
64
+ return :added unless idx
65
+
66
+ if diff_point_phrases[idx]['string'] == head_phrase['string']
67
+ :unmodified
68
+ else
69
+ :modified
70
+ end
71
+ end
72
+
73
+ # identifies phrases in diff point that are not in head ('removed')
74
+ def diff_point_to_head
75
+ return to_enum(__method__) unless block_given?
76
+
77
+ diff_point_phrases.each do |diff_point_phrase|
78
+ idx = head_phrases.find_index do |head_phrase|
79
+ head_phrase['key'] == diff_point_phrase['key']
80
+ end
81
+
82
+ yield(diff_point_phrase, :removed) unless idx
83
+ end
84
+ end
85
+
86
+ def new_hash_of_arrays
87
+ Hash.new { |hash, key| hash[key] = [] }
88
+ end
89
+ end
90
+ end