evrone-ci-router 0.2.0.pre0 → 0.2.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/evrone-ci-router +5 -0
- data/fixtures/jobs_publisher +13 -0
- data/lib/evrone/ci/router.rb +112 -37
- data/lib/evrone/ci/router/build.rb +29 -3
- data/lib/evrone/ci/router/build_matrix.rb +7 -1
- data/lib/evrone/ci/router/cli.rb +67 -0
- data/lib/evrone/ci/router/configuration.rb +7 -1
- data/lib/evrone/ci/router/consumers/builds_consumer.rb +1 -1
- data/lib/evrone/ci/router/initializers/amqp.rb +2 -1
- data/lib/evrone/ci/router/{queue.rb → script_builder.rb} +10 -6
- data/lib/evrone/ci/router/{middleware/travis → script_builder}/env.rb +1 -1
- data/lib/evrone/ci/router/{middleware/travis → script_builder}/ruby.rb +1 -1
- data/lib/evrone/ci/router/{middleware/travis → script_builder}/script.rb +1 -1
- data/lib/evrone/ci/router/travis.rb +2 -2
- data/lib/evrone/ci/router/version.rb +1 -1
- data/spec/lib/build_matrix_spec.rb +13 -2
- data/spec/lib/build_spec.rb +33 -0
- data/spec/lib/configuration_spec.rb +4 -2
- data/spec/lib/router_spec.rb +116 -0
- data/spec/lib/{queue_spec.rb → script_builder_spec.rb} +7 -7
- data/spec/lib/travis_spec.rb +2 -2
- metadata +18 -38
- data/bin/cli +0 -3
- data/bin/git_ssh +0 -3
- data/bin/jobs_publisher +0 -12
- data/bin/workers +0 -13
- data/lib/evrone/ci/router/middleware/create_build_matrix.rb +0 -54
- data/lib/evrone/ci/router/middleware/create_dirs.rb +0 -25
- data/lib/evrone/ci/router/middleware/fetch_commit_info.rb +0 -22
- data/lib/evrone/ci/router/middleware/fetch_source.rb +0 -36
- data/lib/evrone/ci/router/middleware/log_build.rb +0 -24
- data/lib/evrone/ci/router/middleware/update_build_status.rb +0 -75
- data/spec/lib/middleware/create_build_matrix_spec.rb +0 -73
- data/spec/lib/middleware/create_dirs_spec.rb +0 -26
- data/spec/lib/middleware/fetch_commit_info_spec.rb +0 -23
- data/spec/lib/middleware/fetch_source_spec.rb +0 -27
- data/spec/lib/middleware/log_build_spec.rb +0 -14
- data/spec/lib/middleware/update_build_status_spec.rb +0 -70
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edee6c03fb2fd412081abfcc5d0a1728f18a277b
|
4
|
+
data.tar.gz: 8de6b06ce57d13adff8d12ead02ac3a0560436d7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41e077af700321ee78461ced593a229000544e47372fc88dc2c18fa46318de8c1747204460fb990c60297b06edc7f8db43397bc70065cde1cf22a7db2fb23d0a
|
7
|
+
data.tar.gz: 553fe1631f488f868241ff00ca844a20ae049fe054712d47bc66d91854b14e086454c0f2b14a6c02d93afd5ac52081d30cdcff0feb3f9dea1b12a2973deb6546
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require File.expand_path("../../lib/evrone/ci/router", __FILE__)
|
4
|
+
|
5
|
+
require 'evrone/ci/message/testing'
|
6
|
+
|
7
|
+
include Evrone::CI
|
8
|
+
Router::CLI.new
|
9
|
+
|
10
|
+
num = ENV['NUM'] || 1
|
11
|
+
num.to_i.times do
|
12
|
+
Router::BuildsConsumer.publish Message::PerformBuild.test_message
|
13
|
+
end
|
data/lib/evrone/ci/router.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
require 'rubygems'
|
2
|
+
require 'pathname'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'thread'
|
5
|
+
|
2
6
|
require 'bundler'
|
3
7
|
Bundler.require :default
|
4
8
|
|
5
|
-
require 'pathname'
|
6
|
-
|
7
9
|
require File.expand_path("../..", __FILE__) + "/ci/router/ext/string.rb"
|
8
10
|
require File.expand_path("../..", __FILE__) + "/ci/router/ext/array.rb"
|
9
11
|
|
@@ -14,8 +16,9 @@ module Evrone
|
|
14
16
|
autoload :Configuration, File.expand_path("../router/configuration", __FILE__)
|
15
17
|
autoload :Build, File.expand_path("../router/build", __FILE__)
|
16
18
|
autoload :BuildMatrix, File.expand_path("../router/build_matrix", __FILE__)
|
17
|
-
autoload :
|
19
|
+
autoload :ScriptBuilder, File.expand_path("../router/script_builder", __FILE__)
|
18
20
|
autoload :Travis, File.expand_path("../router/travis", __FILE__)
|
21
|
+
autoload :CLI, File.expand_path("../router/cli", __FILE__)
|
19
22
|
|
20
23
|
autoload :BuildLogsConsumer, File.expand_path("../router/consumers/build_logs_consumer", __FILE__)
|
21
24
|
autoload :BuildStatusConsumer, File.expand_path("../router/consumers/build_status_consumer", __FILE__)
|
@@ -28,22 +31,8 @@ module Evrone
|
|
28
31
|
autoload :TraceShCommand, File.expand_path("../router/helper/trace_sh_command", __FILE__)
|
29
32
|
end
|
30
33
|
|
31
|
-
|
32
|
-
|
33
|
-
autoload :Env, File.expand_path("../router/middleware/travis/env", __FILE__)
|
34
|
-
autoload :Ruby, File.expand_path("../router/middleware/travis/ruby", __FILE__)
|
35
|
-
autoload :Script, File.expand_path("../router/middleware/travis/script", __FILE__)
|
36
|
-
end
|
37
|
-
autoload :CreateBuildMatrix, File.expand_path("../router/middleware/create_build_matrix", __FILE__)
|
38
|
-
autoload :FetchSource, File.expand_path("../router/middleware/fetch_source", __FILE__)
|
39
|
-
autoload :FetchCommitInfo, File.expand_path("../router/middleware/fetch_commit_info", __FILE__)
|
40
|
-
autoload :LogBuild, File.expand_path("../router/middleware/log_build", __FILE__)
|
41
|
-
autoload :UpdateBuildStatus, File.expand_path("../router/middleware/update_build_status", __FILE__)
|
42
|
-
autoload :CreateDirs, File.expand_path("../router/middleware/create_dirs", __FILE__)
|
43
|
-
end
|
44
|
-
|
45
|
-
@@root = Pathname.new File.expand_path('../../../..', __FILE__)
|
46
|
-
@@config = Configuration.new
|
34
|
+
@@root = Pathname.new File.expand_path('../../../..', __FILE__)
|
35
|
+
@@config_mutex = Mutex.new
|
47
36
|
|
48
37
|
class << self
|
49
38
|
def root
|
@@ -60,41 +49,127 @@ module Evrone
|
|
60
49
|
end
|
61
50
|
|
62
51
|
def config
|
63
|
-
|
52
|
+
@config ||= begin
|
53
|
+
@@config_mutex.synchronize do
|
54
|
+
Configuration.new
|
55
|
+
end
|
56
|
+
end
|
64
57
|
end
|
65
58
|
|
66
59
|
def reset_config!
|
67
|
-
|
60
|
+
@config = nil
|
68
61
|
end
|
69
|
-
end
|
70
62
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
use Middleware::CreateDirs
|
77
|
-
use Middleware::FetchSource
|
78
|
-
use Middleware::FetchCommitInfo
|
79
|
-
use Middleware::CreateBuildMatrix
|
63
|
+
def initialize!
|
64
|
+
root.join("lib/evrone/ci/router/initializers").children.each do |e|
|
65
|
+
require e
|
66
|
+
end
|
67
|
+
end
|
80
68
|
end
|
81
69
|
|
82
|
-
|
70
|
+
include Helper::Logger
|
71
|
+
include Helper::Config
|
72
|
+
|
73
|
+
attr_reader :build, :path_prefix, :repo_dir, :travis
|
83
74
|
|
84
75
|
def initialize(build, path_prefix)
|
85
76
|
@build = build
|
86
77
|
@path_prefix = Pathname.new(path_prefix).expand_path
|
78
|
+
@repo_dir = @path_prefix.join(config.repo_dir_name)
|
79
|
+
.join(build.message.name)
|
80
|
+
@travis = nil
|
87
81
|
end
|
88
82
|
|
89
83
|
def perform
|
90
|
-
|
91
|
-
|
84
|
+
log_build do
|
85
|
+
update_build_status do
|
86
|
+
create_repo_dir &&
|
87
|
+
fetch_repo &&
|
88
|
+
assign_commit_info &&
|
89
|
+
load_travis &&
|
90
|
+
create_and_delivery_build_matrix
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def create_and_delivery_build_matrix
|
96
|
+
matrix = BuildMatrix.new travis
|
97
|
+
build.matrix = matrix.keys
|
98
|
+
build.jobs_count = matrix.travises.size
|
99
|
+
|
100
|
+
matrix.travises.each_with_index do |travis, idx|
|
101
|
+
number = idx + 1
|
102
|
+
message = build.to_perform_job_message travis, number
|
103
|
+
logger.info "delivery job #{message.id}.#{number} #{travis.to_matrix_s}"
|
104
|
+
JobsConsumer.publish message
|
105
|
+
end
|
106
|
+
|
107
|
+
true
|
92
108
|
end
|
93
109
|
|
110
|
+
def load_travis
|
111
|
+
@travis = Travis.from_file repo_dir.join(".travis.yml")
|
112
|
+
end
|
113
|
+
|
114
|
+
def create_repo_dir
|
115
|
+
FileUtils.mkdir_p(repo_dir) unless repo_dir.directory?
|
116
|
+
true
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_repo
|
120
|
+
scm.fetch == 0
|
121
|
+
end
|
122
|
+
|
123
|
+
def assign_commit_info
|
124
|
+
build.commit_info = scm.commit_info
|
125
|
+
true
|
126
|
+
end
|
127
|
+
|
128
|
+
def log_build
|
129
|
+
logger.tagged("BUILD #{build.message.id}") do
|
130
|
+
logger.info "starting build"
|
131
|
+
rs = yield
|
132
|
+
logger.info "done build"
|
133
|
+
rs
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def update_build_status
|
138
|
+
publish_build_status_message Build::STARTED
|
139
|
+
rs = false
|
140
|
+
begin
|
141
|
+
rs = yield
|
142
|
+
rescue Exception => e
|
143
|
+
logger.error("ERROR: #{e.inspect}\n BACKTRACE:\n#{e.backtrace.map{|i| " #{i}" }.join("\n")}")
|
144
|
+
end
|
145
|
+
|
146
|
+
if rs
|
147
|
+
publish_build_status_message Build::FINISHED
|
148
|
+
else
|
149
|
+
publish_build_status_message Build::FAILED
|
150
|
+
end
|
151
|
+
rs
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
|
156
|
+
def scm
|
157
|
+
@scm ||= SCM::Git.new(
|
158
|
+
build.message.src,
|
159
|
+
build.message.sha,
|
160
|
+
repo_dir,
|
161
|
+
deploy_key: build.message.deploy_key,
|
162
|
+
&build.method(:add_to_output)
|
163
|
+
)
|
164
|
+
end
|
165
|
+
|
166
|
+
def publish_build_status_message(status)
|
167
|
+
message = build.to_build_status_message(status)
|
168
|
+
logger.info "delivered build status #{message.inspect}"
|
169
|
+
BuildStatusConsumer.publish message
|
170
|
+
end
|
171
|
+
|
94
172
|
end
|
95
173
|
end
|
96
174
|
end
|
97
175
|
|
98
|
-
Evrone::CI::Router.root.join("lib/evrone/ci/router/initializers").children.each do |e|
|
99
|
-
require e
|
100
|
-
end
|
@@ -6,6 +6,10 @@ module Evrone
|
|
6
6
|
class Router
|
7
7
|
class Build
|
8
8
|
|
9
|
+
STARTED = 2
|
10
|
+
FINISHED = 3
|
11
|
+
FAILED = 5
|
12
|
+
|
9
13
|
include Router::Helper::Logger
|
10
14
|
include Router::Helper::Config
|
11
15
|
|
@@ -18,7 +22,7 @@ module Evrone
|
|
18
22
|
end
|
19
23
|
|
20
24
|
def to_perform_job_message(travis, job_id)
|
21
|
-
|
25
|
+
script_builder = travis.to_script_builder
|
22
26
|
job_message = Message::PerformJob.new(
|
23
27
|
id: message.id,
|
24
28
|
name: message.name,
|
@@ -27,13 +31,35 @@ module Evrone
|
|
27
31
|
pull_request_id: message.pull_request_id,
|
28
32
|
deploy_key: message.deploy_key,
|
29
33
|
job_id: job_id,
|
30
|
-
before_script:
|
31
|
-
script:
|
34
|
+
before_script: script_builder.to_before_script,
|
35
|
+
script: script_builder.to_script,
|
32
36
|
matrix_keys: travis.matrix_keys,
|
33
37
|
)
|
34
38
|
job_message
|
35
39
|
end
|
36
40
|
|
41
|
+
def to_build_status_message(status)
|
42
|
+
tm = Time.now
|
43
|
+
attributes = {
|
44
|
+
build_id: message.id,
|
45
|
+
status: status,
|
46
|
+
tm: tm.to_i,
|
47
|
+
tm_usec: tm.usec,
|
48
|
+
matrix: matrix || [],
|
49
|
+
jobs_count: jobs_count || 0,
|
50
|
+
}
|
51
|
+
|
52
|
+
if commit_info
|
53
|
+
attributes.merge!(
|
54
|
+
commit_author: commit_info.author,
|
55
|
+
commit_author_email: commit_info.email,
|
56
|
+
commit_sha: commit_info.sha,
|
57
|
+
commit_message: commit_info.message
|
58
|
+
)
|
59
|
+
end
|
60
|
+
Message::BuildStatus.new attributes
|
61
|
+
end
|
62
|
+
|
37
63
|
def add_to_output(str)
|
38
64
|
output << str
|
39
65
|
logger.debug str.strip if logger.level == 0
|
@@ -4,6 +4,7 @@ module Evrone
|
|
4
4
|
class BuildMatrix
|
5
5
|
|
6
6
|
KEYS = (Travis::LANGS + %w{ env }).freeze
|
7
|
+
NOT_MATRIX_KEYS = %w{ script before_script }
|
7
8
|
|
8
9
|
attr_reader :travis
|
9
10
|
|
@@ -17,7 +18,12 @@ module Evrone
|
|
17
18
|
|
18
19
|
def travises
|
19
20
|
attributes_for_new_travises.map do |attrs|
|
20
|
-
Travis.new attrs
|
21
|
+
Travis.new attrs.merge(
|
22
|
+
NOT_MATRIX_KEYS.inject({}) do |a,v|
|
23
|
+
a[v] = travis.public_send(v)
|
24
|
+
a
|
25
|
+
end
|
26
|
+
)
|
21
27
|
end
|
22
28
|
end
|
23
29
|
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'evrone/common/amqp'
|
3
|
+
|
4
|
+
module Evrone
|
5
|
+
module CI
|
6
|
+
class Router
|
7
|
+
class CLI
|
8
|
+
|
9
|
+
include Helper::Config
|
10
|
+
include Helper::Logger
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@options = {}
|
14
|
+
parse!
|
15
|
+
Router.initialize!
|
16
|
+
end
|
17
|
+
|
18
|
+
def run
|
19
|
+
logger.warn "spawn inside #{config.path_prefix}"
|
20
|
+
|
21
|
+
trap('INT') {
|
22
|
+
Thread.new do
|
23
|
+
Evrone::Common::AMQP.shutdown
|
24
|
+
end.join
|
25
|
+
}
|
26
|
+
|
27
|
+
Evrone::Common::AMQP::Supervisor::Threaded.build(
|
28
|
+
Evrone::CI::Router::BuildsConsumer => config.workers,
|
29
|
+
).run
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def parse!
|
35
|
+
OptionParser.new do |opts|
|
36
|
+
opts.banner = "Usage: evrone-ci-router [options]"
|
37
|
+
opts.on("-w", "--workers NUM", "Number of workers, default 1") do |v|
|
38
|
+
@options[:workers] = v.to_i
|
39
|
+
end
|
40
|
+
opts.on("-p", "--path PATH", "Working directory, default current directory") do |v|
|
41
|
+
@options[:path_prefix] = v.to_s
|
42
|
+
end
|
43
|
+
opts.on("-c", "--config FILE", "Path to configuration file") do |v|
|
44
|
+
read_configuration v
|
45
|
+
end
|
46
|
+
end.parse!
|
47
|
+
|
48
|
+
@options.each_pair do |k,v|
|
49
|
+
config.public_send("#{k}=", v)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def read_configuration(file)
|
54
|
+
file = File.expand_path(file)
|
55
|
+
buf = File.read(file)
|
56
|
+
|
57
|
+
buf.split("\n").each do |line|
|
58
|
+
puts line
|
59
|
+
env, value = line.split("=").map(&:strip)
|
60
|
+
ENV[env] = value
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -15,12 +15,18 @@ module Evrone
|
|
15
15
|
define amqp_url: nil,
|
16
16
|
timeout: 30 * 60,
|
17
17
|
logger: Common::TaggedLogging.new(Logger.new STDOUT),
|
18
|
-
repo_dir_name: "repo"
|
18
|
+
repo_dir_name: "repo",
|
19
|
+
workers: 1,
|
20
|
+
path_prefix: nil
|
19
21
|
|
20
22
|
def timeout
|
21
23
|
self[:timeout].to_i
|
22
24
|
end
|
23
25
|
|
26
|
+
def path_prefix
|
27
|
+
self[:path_prefix] || Dir.pwd
|
28
|
+
end
|
29
|
+
|
24
30
|
end
|
25
31
|
end
|
26
32
|
end
|
@@ -16,7 +16,7 @@ module Evrone
|
|
16
16
|
def perform(message)
|
17
17
|
build = Build.new message
|
18
18
|
number = Thread.current[:consumer_id] || 0
|
19
|
-
path_prefix = "/
|
19
|
+
path_prefix = "#{Router.config.path_prefix}/build.#{number}"
|
20
20
|
|
21
21
|
Router.new(build, path_prefix).perform
|
22
22
|
ack!
|
@@ -4,14 +4,18 @@ require 'evrone/ci/common'
|
|
4
4
|
module Evrone
|
5
5
|
module CI
|
6
6
|
class Router
|
7
|
-
class
|
7
|
+
class ScriptBuilder
|
8
|
+
|
9
|
+
autoload :Env, File.expand_path("../script_builder/env", __FILE__)
|
10
|
+
autoload :Ruby, File.expand_path("../script_builder/ruby", __FILE__)
|
11
|
+
autoload :Script, File.expand_path("../script_builder/script", __FILE__)
|
8
12
|
|
9
13
|
include Common::Helper::Middlewares
|
10
14
|
|
11
15
|
middlewares do
|
12
|
-
use
|
13
|
-
use
|
14
|
-
use
|
16
|
+
use ScriptBuilder::Env
|
17
|
+
use ScriptBuilder::Ruby
|
18
|
+
use ScriptBuilder::Script
|
15
19
|
end
|
16
20
|
|
17
21
|
attr_reader :travis
|
@@ -21,7 +25,7 @@ module Evrone
|
|
21
25
|
end
|
22
26
|
|
23
27
|
def to_before_script
|
24
|
-
a = [
|
28
|
+
a = []
|
25
29
|
a += env.init
|
26
30
|
a += env.before_install
|
27
31
|
a += env.install
|
@@ -30,7 +34,7 @@ module Evrone
|
|
30
34
|
end
|
31
35
|
|
32
36
|
def to_script
|
33
|
-
a = [
|
37
|
+
a = []
|
34
38
|
a << env.script
|
35
39
|
a.join("\n")
|
36
40
|
end
|