evrone-ci-router 0.2.0.pre0 → 0.2.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/bin/evrone-ci-router +5 -0
  3. data/fixtures/jobs_publisher +13 -0
  4. data/lib/evrone/ci/router.rb +112 -37
  5. data/lib/evrone/ci/router/build.rb +29 -3
  6. data/lib/evrone/ci/router/build_matrix.rb +7 -1
  7. data/lib/evrone/ci/router/cli.rb +67 -0
  8. data/lib/evrone/ci/router/configuration.rb +7 -1
  9. data/lib/evrone/ci/router/consumers/builds_consumer.rb +1 -1
  10. data/lib/evrone/ci/router/initializers/amqp.rb +2 -1
  11. data/lib/evrone/ci/router/{queue.rb → script_builder.rb} +10 -6
  12. data/lib/evrone/ci/router/{middleware/travis → script_builder}/env.rb +1 -1
  13. data/lib/evrone/ci/router/{middleware/travis → script_builder}/ruby.rb +1 -1
  14. data/lib/evrone/ci/router/{middleware/travis → script_builder}/script.rb +1 -1
  15. data/lib/evrone/ci/router/travis.rb +2 -2
  16. data/lib/evrone/ci/router/version.rb +1 -1
  17. data/spec/lib/build_matrix_spec.rb +13 -2
  18. data/spec/lib/build_spec.rb +33 -0
  19. data/spec/lib/configuration_spec.rb +4 -2
  20. data/spec/lib/router_spec.rb +116 -0
  21. data/spec/lib/{queue_spec.rb → script_builder_spec.rb} +7 -7
  22. data/spec/lib/travis_spec.rb +2 -2
  23. metadata +18 -38
  24. data/bin/cli +0 -3
  25. data/bin/git_ssh +0 -3
  26. data/bin/jobs_publisher +0 -12
  27. data/bin/workers +0 -13
  28. data/lib/evrone/ci/router/middleware/create_build_matrix.rb +0 -54
  29. data/lib/evrone/ci/router/middleware/create_dirs.rb +0 -25
  30. data/lib/evrone/ci/router/middleware/fetch_commit_info.rb +0 -22
  31. data/lib/evrone/ci/router/middleware/fetch_source.rb +0 -36
  32. data/lib/evrone/ci/router/middleware/log_build.rb +0 -24
  33. data/lib/evrone/ci/router/middleware/update_build_status.rb +0 -75
  34. data/spec/lib/middleware/create_build_matrix_spec.rb +0 -73
  35. data/spec/lib/middleware/create_dirs_spec.rb +0 -26
  36. data/spec/lib/middleware/fetch_commit_info_spec.rb +0 -23
  37. data/spec/lib/middleware/fetch_source_spec.rb +0 -27
  38. data/spec/lib/middleware/log_build_spec.rb +0 -14
  39. data/spec/lib/middleware/update_build_status_spec.rb +0 -70
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bfbf13a7842e030d4d20aa825df0c5aede860177
4
- data.tar.gz: da99904e2ddeacee3c87c9b9c56a63e6981dd860
3
+ metadata.gz: edee6c03fb2fd412081abfcc5d0a1728f18a277b
4
+ data.tar.gz: 8de6b06ce57d13adff8d12ead02ac3a0560436d7
5
5
  SHA512:
6
- metadata.gz: d645737756afe9953e85d2054fe436dff8b3dd8b5e7aa209d9f023e2f732c114809db6ded81b28a5f708c7f71405791b69cbdccf3574934e3ee9ff599be9f5e5
7
- data.tar.gz: 9cc5c15fb61e8ffb93809e24c6a979f5218bfca721fceb676456c2b1e6a44831759b38b5ea2359a75753cc773af41ec776ead1df21e9130822376bde7245e3a4
6
+ metadata.gz: 41e077af700321ee78461ced593a229000544e47372fc88dc2c18fa46318de8c1747204460fb990c60297b06edc7f8db43397bc70065cde1cf22a7db2fb23d0a
7
+ data.tar.gz: 553fe1631f488f868241ff00ca844a20ae049fe054712d47bc66d91854b14e086454c0f2b14a6c02d93afd5ac52081d30cdcff0feb3f9dea1b12a2973deb6546
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.expand_path("../../lib/evrone/ci/router", __FILE__)
4
+
5
+ Evrone::CI::Router::CLI.new.run
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.expand_path("../../lib/evrone/ci/router", __FILE__)
4
+
5
+ require 'evrone/ci/message/testing'
6
+
7
+ include Evrone::CI
8
+ Router::CLI.new
9
+
10
+ num = ENV['NUM'] || 1
11
+ num.to_i.times do
12
+ Router::BuildsConsumer.publish Message::PerformBuild.test_message
13
+ end
@@ -1,9 +1,11 @@
1
1
  require 'rubygems'
2
+ require 'pathname'
3
+ require 'fileutils'
4
+ require 'thread'
5
+
2
6
  require 'bundler'
3
7
  Bundler.require :default
4
8
 
5
- require 'pathname'
6
-
7
9
  require File.expand_path("../..", __FILE__) + "/ci/router/ext/string.rb"
8
10
  require File.expand_path("../..", __FILE__) + "/ci/router/ext/array.rb"
9
11
 
@@ -14,8 +16,9 @@ module Evrone
14
16
  autoload :Configuration, File.expand_path("../router/configuration", __FILE__)
15
17
  autoload :Build, File.expand_path("../router/build", __FILE__)
16
18
  autoload :BuildMatrix, File.expand_path("../router/build_matrix", __FILE__)
17
- autoload :Queue, File.expand_path("../router/queue", __FILE__)
19
+ autoload :ScriptBuilder, File.expand_path("../router/script_builder", __FILE__)
18
20
  autoload :Travis, File.expand_path("../router/travis", __FILE__)
21
+ autoload :CLI, File.expand_path("../router/cli", __FILE__)
19
22
 
20
23
  autoload :BuildLogsConsumer, File.expand_path("../router/consumers/build_logs_consumer", __FILE__)
21
24
  autoload :BuildStatusConsumer, File.expand_path("../router/consumers/build_status_consumer", __FILE__)
@@ -28,22 +31,8 @@ module Evrone
28
31
  autoload :TraceShCommand, File.expand_path("../router/helper/trace_sh_command", __FILE__)
29
32
  end
30
33
 
31
- module Middleware
32
- module Travis
33
- autoload :Env, File.expand_path("../router/middleware/travis/env", __FILE__)
34
- autoload :Ruby, File.expand_path("../router/middleware/travis/ruby", __FILE__)
35
- autoload :Script, File.expand_path("../router/middleware/travis/script", __FILE__)
36
- end
37
- autoload :CreateBuildMatrix, File.expand_path("../router/middleware/create_build_matrix", __FILE__)
38
- autoload :FetchSource, File.expand_path("../router/middleware/fetch_source", __FILE__)
39
- autoload :FetchCommitInfo, File.expand_path("../router/middleware/fetch_commit_info", __FILE__)
40
- autoload :LogBuild, File.expand_path("../router/middleware/log_build", __FILE__)
41
- autoload :UpdateBuildStatus, File.expand_path("../router/middleware/update_build_status", __FILE__)
42
- autoload :CreateDirs, File.expand_path("../router/middleware/create_dirs", __FILE__)
43
- end
44
-
45
- @@root = Pathname.new File.expand_path('../../../..', __FILE__)
46
- @@config = Configuration.new
34
+ @@root = Pathname.new File.expand_path('../../../..', __FILE__)
35
+ @@config_mutex = Mutex.new
47
36
 
48
37
  class << self
49
38
  def root
@@ -60,41 +49,127 @@ module Evrone
60
49
  end
61
50
 
62
51
  def config
63
- @@config
52
+ @config ||= begin
53
+ @@config_mutex.synchronize do
54
+ Configuration.new
55
+ end
56
+ end
64
57
  end
65
58
 
66
59
  def reset_config!
67
- @@config = Configuration.new
60
+ @config = nil
68
61
  end
69
- end
70
62
 
71
- include Common::Helper::Middlewares
72
-
73
- middlewares do
74
- use Middleware::LogBuild
75
- use Middleware::UpdateBuildStatus
76
- use Middleware::CreateDirs
77
- use Middleware::FetchSource
78
- use Middleware::FetchCommitInfo
79
- use Middleware::CreateBuildMatrix
63
+ def initialize!
64
+ root.join("lib/evrone/ci/router/initializers").children.each do |e|
65
+ require e
66
+ end
67
+ end
80
68
  end
81
69
 
82
- attr_reader :build, :path_prefix
70
+ include Helper::Logger
71
+ include Helper::Config
72
+
73
+ attr_reader :build, :path_prefix, :repo_dir, :travis
83
74
 
84
75
  def initialize(build, path_prefix)
85
76
  @build = build
86
77
  @path_prefix = Pathname.new(path_prefix).expand_path
78
+ @repo_dir = @path_prefix.join(config.repo_dir_name)
79
+ .join(build.message.name)
80
+ @travis = nil
87
81
  end
88
82
 
89
83
  def perform
90
- env = OpenStruct.new build: build, path_prefix: path_prefix
91
- run_middlewares(env) { |_| 0 }
84
+ log_build do
85
+ update_build_status do
86
+ create_repo_dir &&
87
+ fetch_repo &&
88
+ assign_commit_info &&
89
+ load_travis &&
90
+ create_and_delivery_build_matrix
91
+ end
92
+ end
93
+ end
94
+
95
+ def create_and_delivery_build_matrix
96
+ matrix = BuildMatrix.new travis
97
+ build.matrix = matrix.keys
98
+ build.jobs_count = matrix.travises.size
99
+
100
+ matrix.travises.each_with_index do |travis, idx|
101
+ number = idx + 1
102
+ message = build.to_perform_job_message travis, number
103
+ logger.info "delivery job #{message.id}.#{number} #{travis.to_matrix_s}"
104
+ JobsConsumer.publish message
105
+ end
106
+
107
+ true
92
108
  end
93
109
 
110
+ def load_travis
111
+ @travis = Travis.from_file repo_dir.join(".travis.yml")
112
+ end
113
+
114
+ def create_repo_dir
115
+ FileUtils.mkdir_p(repo_dir) unless repo_dir.directory?
116
+ true
117
+ end
118
+
119
+ def fetch_repo
120
+ scm.fetch == 0
121
+ end
122
+
123
+ def assign_commit_info
124
+ build.commit_info = scm.commit_info
125
+ true
126
+ end
127
+
128
+ def log_build
129
+ logger.tagged("BUILD #{build.message.id}") do
130
+ logger.info "starting build"
131
+ rs = yield
132
+ logger.info "done build"
133
+ rs
134
+ end
135
+ end
136
+
137
+ def update_build_status
138
+ publish_build_status_message Build::STARTED
139
+ rs = false
140
+ begin
141
+ rs = yield
142
+ rescue Exception => e
143
+ logger.error("ERROR: #{e.inspect}\n BACKTRACE:\n#{e.backtrace.map{|i| " #{i}" }.join("\n")}")
144
+ end
145
+
146
+ if rs
147
+ publish_build_status_message Build::FINISHED
148
+ else
149
+ publish_build_status_message Build::FAILED
150
+ end
151
+ rs
152
+ end
153
+
154
+ private
155
+
156
+ def scm
157
+ @scm ||= SCM::Git.new(
158
+ build.message.src,
159
+ build.message.sha,
160
+ repo_dir,
161
+ deploy_key: build.message.deploy_key,
162
+ &build.method(:add_to_output)
163
+ )
164
+ end
165
+
166
+ def publish_build_status_message(status)
167
+ message = build.to_build_status_message(status)
168
+ logger.info "delivered build status #{message.inspect}"
169
+ BuildStatusConsumer.publish message
170
+ end
171
+
94
172
  end
95
173
  end
96
174
  end
97
175
 
98
- Evrone::CI::Router.root.join("lib/evrone/ci/router/initializers").children.each do |e|
99
- require e
100
- end
@@ -6,6 +6,10 @@ module Evrone
6
6
  class Router
7
7
  class Build
8
8
 
9
+ STARTED = 2
10
+ FINISHED = 3
11
+ FAILED = 5
12
+
9
13
  include Router::Helper::Logger
10
14
  include Router::Helper::Config
11
15
 
@@ -18,7 +22,7 @@ module Evrone
18
22
  end
19
23
 
20
24
  def to_perform_job_message(travis, job_id)
21
- queue = travis.to_queue
25
+ script_builder = travis.to_script_builder
22
26
  job_message = Message::PerformJob.new(
23
27
  id: message.id,
24
28
  name: message.name,
@@ -27,13 +31,35 @@ module Evrone
27
31
  pull_request_id: message.pull_request_id,
28
32
  deploy_key: message.deploy_key,
29
33
  job_id: job_id,
30
- before_script: queue.to_before_script,
31
- script: queue.to_script,
34
+ before_script: script_builder.to_before_script,
35
+ script: script_builder.to_script,
32
36
  matrix_keys: travis.matrix_keys,
33
37
  )
34
38
  job_message
35
39
  end
36
40
 
41
+ def to_build_status_message(status)
42
+ tm = Time.now
43
+ attributes = {
44
+ build_id: message.id,
45
+ status: status,
46
+ tm: tm.to_i,
47
+ tm_usec: tm.usec,
48
+ matrix: matrix || [],
49
+ jobs_count: jobs_count || 0,
50
+ }
51
+
52
+ if commit_info
53
+ attributes.merge!(
54
+ commit_author: commit_info.author,
55
+ commit_author_email: commit_info.email,
56
+ commit_sha: commit_info.sha,
57
+ commit_message: commit_info.message
58
+ )
59
+ end
60
+ Message::BuildStatus.new attributes
61
+ end
62
+
37
63
  def add_to_output(str)
38
64
  output << str
39
65
  logger.debug str.strip if logger.level == 0
@@ -4,6 +4,7 @@ module Evrone
4
4
  class BuildMatrix
5
5
 
6
6
  KEYS = (Travis::LANGS + %w{ env }).freeze
7
+ NOT_MATRIX_KEYS = %w{ script before_script }
7
8
 
8
9
  attr_reader :travis
9
10
 
@@ -17,7 +18,12 @@ module Evrone
17
18
 
18
19
  def travises
19
20
  attributes_for_new_travises.map do |attrs|
20
- Travis.new attrs
21
+ Travis.new attrs.merge(
22
+ NOT_MATRIX_KEYS.inject({}) do |a,v|
23
+ a[v] = travis.public_send(v)
24
+ a
25
+ end
26
+ )
21
27
  end
22
28
  end
23
29
 
@@ -0,0 +1,67 @@
1
+ require 'optparse'
2
+ require 'evrone/common/amqp'
3
+
4
+ module Evrone
5
+ module CI
6
+ class Router
7
+ class CLI
8
+
9
+ include Helper::Config
10
+ include Helper::Logger
11
+
12
+ def initialize
13
+ @options = {}
14
+ parse!
15
+ Router.initialize!
16
+ end
17
+
18
+ def run
19
+ logger.warn "spawn inside #{config.path_prefix}"
20
+
21
+ trap('INT') {
22
+ Thread.new do
23
+ Evrone::Common::AMQP.shutdown
24
+ end.join
25
+ }
26
+
27
+ Evrone::Common::AMQP::Supervisor::Threaded.build(
28
+ Evrone::CI::Router::BuildsConsumer => config.workers,
29
+ ).run
30
+ end
31
+
32
+ private
33
+
34
+ def parse!
35
+ OptionParser.new do |opts|
36
+ opts.banner = "Usage: evrone-ci-router [options]"
37
+ opts.on("-w", "--workers NUM", "Number of workers, default 1") do |v|
38
+ @options[:workers] = v.to_i
39
+ end
40
+ opts.on("-p", "--path PATH", "Working directory, default current directory") do |v|
41
+ @options[:path_prefix] = v.to_s
42
+ end
43
+ opts.on("-c", "--config FILE", "Path to configuration file") do |v|
44
+ read_configuration v
45
+ end
46
+ end.parse!
47
+
48
+ @options.each_pair do |k,v|
49
+ config.public_send("#{k}=", v)
50
+ end
51
+ end
52
+
53
+ def read_configuration(file)
54
+ file = File.expand_path(file)
55
+ buf = File.read(file)
56
+
57
+ buf.split("\n").each do |line|
58
+ puts line
59
+ env, value = line.split("=").map(&:strip)
60
+ ENV[env] = value
61
+ end
62
+ end
63
+
64
+ end
65
+ end
66
+ end
67
+ end
@@ -15,12 +15,18 @@ module Evrone
15
15
  define amqp_url: nil,
16
16
  timeout: 30 * 60,
17
17
  logger: Common::TaggedLogging.new(Logger.new STDOUT),
18
- repo_dir_name: "repo"
18
+ repo_dir_name: "repo",
19
+ workers: 1,
20
+ path_prefix: nil
19
21
 
20
22
  def timeout
21
23
  self[:timeout].to_i
22
24
  end
23
25
 
26
+ def path_prefix
27
+ self[:path_prefix] || Dir.pwd
28
+ end
29
+
24
30
  end
25
31
  end
26
32
  end
@@ -16,7 +16,7 @@ module Evrone
16
16
  def perform(message)
17
17
  build = Build.new message
18
18
  number = Thread.current[:consumer_id] || 0
19
- path_prefix = "/tmp/.test/build.#{number}"
19
+ path_prefix = "#{Router.config.path_prefix}/build.#{number}"
20
20
 
21
21
  Router.new(build, path_prefix).perform
22
22
  ack!
@@ -59,5 +59,6 @@ Evrone::Common::AMQP.configure do |c|
59
59
  end
60
60
 
61
61
  c.content_type = 'application/x-protobuf'
62
- c.logger = nil
62
+ c.logger = nil
63
+ c.url = Evrone::CI::Router.config.amqp_url
63
64
  end
@@ -4,14 +4,18 @@ require 'evrone/ci/common'
4
4
  module Evrone
5
5
  module CI
6
6
  class Router
7
- class Queue
7
+ class ScriptBuilder
8
+
9
+ autoload :Env, File.expand_path("../script_builder/env", __FILE__)
10
+ autoload :Ruby, File.expand_path("../script_builder/ruby", __FILE__)
11
+ autoload :Script, File.expand_path("../script_builder/script", __FILE__)
8
12
 
9
13
  include Common::Helper::Middlewares
10
14
 
11
15
  middlewares do
12
- use Middleware::Travis::Env
13
- use Middleware::Travis::Ruby
14
- use Middleware::Travis::Script
16
+ use ScriptBuilder::Env
17
+ use ScriptBuilder::Ruby
18
+ use ScriptBuilder::Script
15
19
  end
16
20
 
17
21
  attr_reader :travis
@@ -21,7 +25,7 @@ module Evrone
21
25
  end
22
26
 
23
27
  def to_before_script
24
- a = ["set -e"]
28
+ a = []
25
29
  a += env.init
26
30
  a += env.before_install
27
31
  a += env.install
@@ -30,7 +34,7 @@ module Evrone
30
34
  end
31
35
 
32
36
  def to_script
33
- a = ["set -e"]
37
+ a = []
34
38
  a << env.script
35
39
  a.join("\n")
36
40
  end