evrone-ci-router 0.2.0.pre0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +7 -0
- data/.rspec +3 -0
- data/Gemfile +7 -0
- data/README.md +1 -0
- data/Rakefile +22 -0
- data/bin/cli +3 -0
- data/bin/git_ssh +3 -0
- data/bin/jobs_publisher +12 -0
- data/bin/workers +13 -0
- data/evrone-ci-router.gemspec +30 -0
- data/fixtures/travis.yml +5 -0
- data/lib/evrone/ci/router.rb +100 -0
- data/lib/evrone/ci/router/build.rb +60 -0
- data/lib/evrone/ci/router/build_matrix.rb +89 -0
- data/lib/evrone/ci/router/configuration.rb +27 -0
- data/lib/evrone/ci/router/consumers/build_logs_consumer.rb +15 -0
- data/lib/evrone/ci/router/consumers/build_status_consumer.rb +15 -0
- data/lib/evrone/ci/router/consumers/builds_consumer.rb +28 -0
- data/lib/evrone/ci/router/consumers/jobs_consumer.rb +15 -0
- data/lib/evrone/ci/router/ext/array.rb +5 -0
- data/lib/evrone/ci/router/ext/string.rb +9 -0
- data/lib/evrone/ci/router/helper/config.rb +13 -0
- data/lib/evrone/ci/router/helper/logger.rb +13 -0
- data/lib/evrone/ci/router/helper/trace_sh_command.rb +14 -0
- data/lib/evrone/ci/router/initializers/amqp.rb +63 -0
- data/lib/evrone/ci/router/middleware/create_build_matrix.rb +54 -0
- data/lib/evrone/ci/router/middleware/create_dirs.rb +25 -0
- data/lib/evrone/ci/router/middleware/fetch_commit_info.rb +22 -0
- data/lib/evrone/ci/router/middleware/fetch_source.rb +36 -0
- data/lib/evrone/ci/router/middleware/log_build.rb +24 -0
- data/lib/evrone/ci/router/middleware/travis/env.rb +21 -0
- data/lib/evrone/ci/router/middleware/travis/ruby.rb +55 -0
- data/lib/evrone/ci/router/middleware/travis/script.rb +25 -0
- data/lib/evrone/ci/router/middleware/update_build_status.rb +75 -0
- data/lib/evrone/ci/router/queue.rb +58 -0
- data/lib/evrone/ci/router/travis.rb +104 -0
- data/lib/evrone/ci/router/travis/serializable.rb +45 -0
- data/lib/evrone/ci/router/version.rb +7 -0
- data/spec/lib/build_matrix_spec.rb +145 -0
- data/spec/lib/build_spec.rb +75 -0
- data/spec/lib/configuration_spec.rb +22 -0
- data/spec/lib/middleware/create_build_matrix_spec.rb +73 -0
- data/spec/lib/middleware/create_dirs_spec.rb +26 -0
- data/spec/lib/middleware/fetch_commit_info_spec.rb +23 -0
- data/spec/lib/middleware/fetch_source_spec.rb +27 -0
- data/spec/lib/middleware/log_build_spec.rb +14 -0
- data/spec/lib/middleware/update_build_status_spec.rb +70 -0
- data/spec/lib/queue_spec.rb +55 -0
- data/spec/lib/travis_spec.rb +182 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/support/create.rb +47 -0
- data/spec/support/fixture.rb +7 -0
- data/spec/support/shared_examples/update_build_status_message.rb +5 -0
- metadata +228 -0
@@ -0,0 +1,75 @@
|
|
1
|
+
module Evrone
|
2
|
+
module CI
|
3
|
+
class Router
|
4
|
+
module Middleware
|
5
|
+
|
6
|
+
UpdateBuildStatus = Struct.new(:app) do
|
7
|
+
|
8
|
+
include Helper::Logger
|
9
|
+
|
10
|
+
STARTED = 2
|
11
|
+
FINISHED = 3
|
12
|
+
BROKEN = 4
|
13
|
+
FAILED = 5
|
14
|
+
|
15
|
+
def call(env)
|
16
|
+
|
17
|
+
update_status env.build, STARTED
|
18
|
+
rs = -1
|
19
|
+
begin
|
20
|
+
rs = app.call env
|
21
|
+
rescue Exception => e
|
22
|
+
logger.error("ERROR: #{e.inspect}\n BACKTRACE:\n#{e.backtrace.map{|i| " #{i}" }.join("\n")}")
|
23
|
+
end
|
24
|
+
|
25
|
+
case
|
26
|
+
when rs == 0
|
27
|
+
update_status env.build, FINISHED
|
28
|
+
when rs > 0
|
29
|
+
update_status env.build, BROKEN
|
30
|
+
when rs < 0
|
31
|
+
update_status env.build, FAILED
|
32
|
+
end
|
33
|
+
|
34
|
+
rs
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def update_status(build, status)
|
40
|
+
publish_status create_message(build, status), status
|
41
|
+
end
|
42
|
+
|
43
|
+
def create_message(build, status)
|
44
|
+
tm = Time.now
|
45
|
+
attributes = {
|
46
|
+
build_id: build.message.id,
|
47
|
+
status: status,
|
48
|
+
tm: tm.to_i,
|
49
|
+
tm_usec: tm.usec,
|
50
|
+
matrix: build.matrix || [],
|
51
|
+
jobs_count: build.jobs_count || 0,
|
52
|
+
}
|
53
|
+
|
54
|
+
if build.commit_info
|
55
|
+
attributes.merge!(
|
56
|
+
commit_author: build.commit_info.author,
|
57
|
+
commit_author_email: build.commit_info.email,
|
58
|
+
commit_sha: build.commit_info.sha,
|
59
|
+
commit_message: build.commit_info.message
|
60
|
+
)
|
61
|
+
end
|
62
|
+
Message::BuildStatus.new attributes
|
63
|
+
end
|
64
|
+
|
65
|
+
def publish_status(message, status)
|
66
|
+
logger.info "delivered build status #{message.inspect}"
|
67
|
+
BuildStatusConsumer.publish message
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'shellwords'
|
2
|
+
require 'evrone/ci/common'
|
3
|
+
|
4
|
+
module Evrone
|
5
|
+
module CI
|
6
|
+
class Router
|
7
|
+
class Queue
|
8
|
+
|
9
|
+
include Common::Helper::Middlewares
|
10
|
+
|
11
|
+
middlewares do
|
12
|
+
use Middleware::Travis::Env
|
13
|
+
use Middleware::Travis::Ruby
|
14
|
+
use Middleware::Travis::Script
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :travis
|
18
|
+
|
19
|
+
def initialize(travis)
|
20
|
+
@travis = travis
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_before_script
|
24
|
+
a = ["set -e"]
|
25
|
+
a += env.init
|
26
|
+
a += env.before_install
|
27
|
+
a += env.install
|
28
|
+
a += env.before_script
|
29
|
+
a.join("\n")
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_script
|
33
|
+
a = ["set -e"]
|
34
|
+
a << env.script
|
35
|
+
a.join("\n")
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def env
|
41
|
+
@env ||= run_middlewares(default_env) {|_| _ }
|
42
|
+
end
|
43
|
+
|
44
|
+
def default_env
|
45
|
+
OpenStruct.new(
|
46
|
+
init: [],
|
47
|
+
before_install: [],
|
48
|
+
install: [],
|
49
|
+
before_script: [],
|
50
|
+
script: [],
|
51
|
+
travis: travis
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
require File.expand_path("../travis/serializable", __FILE__)
|
2
|
+
|
3
|
+
module Evrone
|
4
|
+
module CI
|
5
|
+
class Router
|
6
|
+
class Travis
|
7
|
+
|
8
|
+
include Travis::Serializable
|
9
|
+
|
10
|
+
LANGS = %w{ rvm scala java go }.freeze
|
11
|
+
KEYS = %w{ before_script script }.freeze
|
12
|
+
AS_ARRAY = (KEYS + LANGS).freeze
|
13
|
+
|
14
|
+
attr_reader :attributes
|
15
|
+
alias_method :to_hash, :attributes
|
16
|
+
|
17
|
+
def initialize(attrs = {})
|
18
|
+
@attributes = normalize_attributes attrs
|
19
|
+
end
|
20
|
+
|
21
|
+
def [](val)
|
22
|
+
public_send(val)
|
23
|
+
end
|
24
|
+
|
25
|
+
def matrix_keys
|
26
|
+
@matrix_keys ||=
|
27
|
+
BuildMatrix::KEYS.inject([]) do |a,k|
|
28
|
+
val = send(k)
|
29
|
+
unless val.empty?
|
30
|
+
a << val.map{|v| "#{k}:#{v}" }
|
31
|
+
end
|
32
|
+
a
|
33
|
+
end.flatten.sort
|
34
|
+
end
|
35
|
+
|
36
|
+
def to_matrix_s
|
37
|
+
@to_matrix_s ||= matrix_keys.join(", ")
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_queue
|
41
|
+
Queue.new(self)
|
42
|
+
end
|
43
|
+
|
44
|
+
def env
|
45
|
+
attributes["env"]["matrix"]
|
46
|
+
end
|
47
|
+
|
48
|
+
def global_env
|
49
|
+
attributes["env"]["global"]
|
50
|
+
end
|
51
|
+
|
52
|
+
AS_ARRAY.each do |m|
|
53
|
+
define_method m do
|
54
|
+
@attributes[m] || []
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def merge(attrs = {})
|
59
|
+
self.class.from_attributes self.attributes.merge(attrs)
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def normalize_attributes(attributes)
|
65
|
+
attributes = attributes.inject({}) do |a,row|
|
66
|
+
k,v = row
|
67
|
+
if AS_ARRAY.include?(k.to_s)
|
68
|
+
v = Array(v)
|
69
|
+
end
|
70
|
+
a[k.to_s] = v
|
71
|
+
a
|
72
|
+
end
|
73
|
+
normalize_env_attribute attributes
|
74
|
+
end
|
75
|
+
|
76
|
+
def normalize_env_attribute(attributes)
|
77
|
+
env = (attributes['env'] || {}) .dup
|
78
|
+
case env
|
79
|
+
when Hash
|
80
|
+
attributes["env"] = {
|
81
|
+
"matrix" => Array(env['matrix']),
|
82
|
+
"global" => Array(env['global'])
|
83
|
+
}
|
84
|
+
else
|
85
|
+
attributes['env'] = {
|
86
|
+
"matrix" => Array(env).map(&:to_s),
|
87
|
+
"global" => []
|
88
|
+
}
|
89
|
+
end
|
90
|
+
freeze_normalized_attributes attributes
|
91
|
+
end
|
92
|
+
|
93
|
+
def freeze_normalized_attributes(attributes)
|
94
|
+
attributes.freeze
|
95
|
+
attributes['env'].freeze
|
96
|
+
attributes['env']['global'].freeze
|
97
|
+
attributes['env']['matrix'].freeze
|
98
|
+
attributes
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Evrone
|
5
|
+
module CI
|
6
|
+
class Router
|
7
|
+
class Travis
|
8
|
+
|
9
|
+
module Serializable
|
10
|
+
|
11
|
+
def self.included(base)
|
12
|
+
base.extend ClassMethods
|
13
|
+
end
|
14
|
+
|
15
|
+
def to_yaml
|
16
|
+
YAML.dump(attributes)
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_hash
|
20
|
+
attributes
|
21
|
+
end
|
22
|
+
|
23
|
+
module ClassMethods
|
24
|
+
|
25
|
+
def from_file(file)
|
26
|
+
if File.readable? file
|
27
|
+
from_yaml File.read(file)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def from_yaml(yaml)
|
32
|
+
from_attributes YAML.load(yaml)
|
33
|
+
end
|
34
|
+
|
35
|
+
def from_attributes(attrs)
|
36
|
+
Travis.new attrs
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
describe Evrone::CI::Router::BuildMatrix do
|
5
|
+
let(:attributes) { {
|
6
|
+
env: %w{ FOO=1 BAR=2 },
|
7
|
+
rvm: %w{ 1.8.7 1.9.3 2.0.0 },
|
8
|
+
scala: %w{ 2.9.2 2.10.1 }
|
9
|
+
} }
|
10
|
+
let(:travis) { create :travis, attributes: attributes }
|
11
|
+
let(:matrix) { described_class.new travis }
|
12
|
+
|
13
|
+
subject { matrix }
|
14
|
+
|
15
|
+
context "just created" do
|
16
|
+
its(:travis) { should eq travis }
|
17
|
+
end
|
18
|
+
|
19
|
+
context "keys" do
|
20
|
+
subject { matrix.keys }
|
21
|
+
it { should eq %w{ env rvm scala } }
|
22
|
+
context "without matrix" do
|
23
|
+
let(:attributes) { {} }
|
24
|
+
|
25
|
+
it { should eq [] }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'travises' do
|
30
|
+
context "values" do
|
31
|
+
subject { matrix.travises }
|
32
|
+
|
33
|
+
it { should have(12).items }
|
34
|
+
|
35
|
+
context "attributes" do
|
36
|
+
subject { matrix.travises.map(&:to_matrix_s) }
|
37
|
+
|
38
|
+
it do
|
39
|
+
should eq [
|
40
|
+
"env:BAR=2, rvm:1.8.7, scala:2.10.1",
|
41
|
+
"env:FOO=1, rvm:1.8.7, scala:2.10.1",
|
42
|
+
"env:BAR=2, rvm:1.8.7, scala:2.9.2",
|
43
|
+
"env:FOO=1, rvm:1.8.7, scala:2.9.2",
|
44
|
+
"env:BAR=2, rvm:1.9.3, scala:2.10.1",
|
45
|
+
"env:FOO=1, rvm:1.9.3, scala:2.10.1",
|
46
|
+
"env:BAR=2, rvm:1.9.3, scala:2.9.2",
|
47
|
+
"env:FOO=1, rvm:1.9.3, scala:2.9.2",
|
48
|
+
"env:BAR=2, rvm:2.0.0, scala:2.10.1",
|
49
|
+
"env:FOO=1, rvm:2.0.0, scala:2.10.1",
|
50
|
+
"env:BAR=2, rvm:2.0.0, scala:2.9.2",
|
51
|
+
"env:FOO=1, rvm:2.0.0, scala:2.9.2"
|
52
|
+
]
|
53
|
+
end
|
54
|
+
|
55
|
+
context "without matrix" do
|
56
|
+
let(:attributes) { {
|
57
|
+
rvm: %w{ 2.0.0 },
|
58
|
+
} }
|
59
|
+
|
60
|
+
it { should eq ['rvm:2.0.0'] }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'attributes_for_new_travises' do
|
67
|
+
subject { matrix.attributes_for_new_travises }
|
68
|
+
|
69
|
+
it { should have(12).items }
|
70
|
+
|
71
|
+
its(:first) { should eq("rvm" => "1.8.7",
|
72
|
+
"scala" => "2.10.1",
|
73
|
+
"env" => "BAR=2") }
|
74
|
+
its(:last) { should eq("rvm" => "2.0.0",
|
75
|
+
"scala" => "2.9.2",
|
76
|
+
"env" => "FOO=1") }
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'extract_pair_of_key_and_values' do
|
80
|
+
subject { matrix.extract_pair_of_key_and_values }
|
81
|
+
it {
|
82
|
+
should eq [
|
83
|
+
["rvm", %w{ 1.8.7 1.9.3 2.0.0 }],
|
84
|
+
["scala", %w{ 2.9.2 2.10.1 }],
|
85
|
+
["env", %w{ FOO=1 BAR=2 }]
|
86
|
+
]
|
87
|
+
}
|
88
|
+
end
|
89
|
+
|
90
|
+
context "permutate_and_build_values" do
|
91
|
+
subject { format_values matrix.permutate_and_build_values }
|
92
|
+
let(:expected) { [
|
93
|
+
%w{env:BAR=2 rvm:1.8.7 scala:2.10.1},
|
94
|
+
%w{env:BAR=2 rvm:1.8.7 scala:2.9.2},
|
95
|
+
%w{env:BAR=2 rvm:1.9.3 scala:2.10.1},
|
96
|
+
%w{env:BAR=2 rvm:1.9.3 scala:2.9.2},
|
97
|
+
%w{env:BAR=2 rvm:2.0.0 scala:2.10.1},
|
98
|
+
%w{env:BAR=2 rvm:2.0.0 scala:2.9.2},
|
99
|
+
%w{env:FOO=1 rvm:1.8.7 scala:2.10.1},
|
100
|
+
%w{env:FOO=1 rvm:1.8.7 scala:2.9.2},
|
101
|
+
%w{env:FOO=1 rvm:1.9.3 scala:2.10.1},
|
102
|
+
%w{env:FOO=1 rvm:1.9.3 scala:2.9.2},
|
103
|
+
%w{env:FOO=1 rvm:2.0.0 scala:2.10.1},
|
104
|
+
%w{env:FOO=1 rvm:2.0.0 scala:2.9.2},
|
105
|
+
] }
|
106
|
+
|
107
|
+
it { should eq expected }
|
108
|
+
|
109
|
+
context "with empty keys" do
|
110
|
+
let(:attributes) { {
|
111
|
+
env: %w{ FOO=1 BAR=2 },
|
112
|
+
rvm: %w{ 1.8.7 1.9.3 2.0.0 },
|
113
|
+
scala: %w{ 2.9.2 2.10.1 },
|
114
|
+
java: [],
|
115
|
+
go: nil
|
116
|
+
} }
|
117
|
+
it { should eq expected }
|
118
|
+
end
|
119
|
+
|
120
|
+
context "with one key" do
|
121
|
+
let(:attributes) { {
|
122
|
+
rvm: %w{ 1.9.3 2.0.0 },
|
123
|
+
} }
|
124
|
+
let(:expected) {[
|
125
|
+
%w{ rvm:1.9.3 },
|
126
|
+
%w{ rvm:2.0.0 }
|
127
|
+
]}
|
128
|
+
it { should eq expected }
|
129
|
+
end
|
130
|
+
|
131
|
+
context "without matrix" do
|
132
|
+
let(:attributes) { {
|
133
|
+
rvm: %w{ 2.0.0 },
|
134
|
+
} }
|
135
|
+
let(:expected) {[
|
136
|
+
%w{ rvm:2.0.0 }
|
137
|
+
]}
|
138
|
+
it { should eq expected }
|
139
|
+
end
|
140
|
+
|
141
|
+
def format_values(values)
|
142
|
+
values.map{|i| i.map(&:to_s).sort }.sort
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Evrone::CI::Router::Build do
|
4
|
+
let(:msg) { create :message, 'PerformBuild' }
|
5
|
+
let(:build) { described_class.new msg }
|
6
|
+
|
7
|
+
subject { build }
|
8
|
+
|
9
|
+
context "just created" do
|
10
|
+
its(:message) { should eq msg }
|
11
|
+
its(:output) { should eq '' }
|
12
|
+
end
|
13
|
+
|
14
|
+
context "create_build_log_message" do
|
15
|
+
let(:tm) { Time.new(2012, 12, 10, 15, 45) }
|
16
|
+
let(:data) { 'log' }
|
17
|
+
subject { build.create_build_log_message data }
|
18
|
+
|
19
|
+
before do
|
20
|
+
mock(Time).now { tm }
|
21
|
+
end
|
22
|
+
|
23
|
+
it { should be_an_instance_of(Evrone::CI::Message::BuildLog) }
|
24
|
+
its(:build_id) { should eq build.message.id }
|
25
|
+
its(:tm) { should eq tm.to_i }
|
26
|
+
its(:log) { should eq data }
|
27
|
+
end
|
28
|
+
|
29
|
+
context "add_to_output" do
|
30
|
+
let(:data) { 'data' }
|
31
|
+
let(:messages) { Evrone::CI::Router::BuildLogsConsumer.messages }
|
32
|
+
subject { build.add_to_output(data) ; build }
|
33
|
+
|
34
|
+
its(:output) { should eq data }
|
35
|
+
it "should delivery message" do
|
36
|
+
expect {
|
37
|
+
subject
|
38
|
+
}.to change(messages, :size).by(1)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context "add_command_to_output" do
|
43
|
+
let(:data) { 'data' }
|
44
|
+
let(:messages) { Evrone::CI::Router::BuildLogsConsumer.messages }
|
45
|
+
subject { build.add_command_to_output(data) ; build }
|
46
|
+
|
47
|
+
its(:output) { should eq "$ #{data}\n" }
|
48
|
+
it "should delivery message" do
|
49
|
+
expect {
|
50
|
+
subject
|
51
|
+
}.to change(messages, :size).by(1)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context ".to_perform_job_message" do
|
56
|
+
let(:travis) { create :travis }
|
57
|
+
let(:job_id) { 2 }
|
58
|
+
subject { build.to_perform_job_message travis, job_id }
|
59
|
+
|
60
|
+
it { should be_an_instance_of Evrone::CI::Message::PerformJob }
|
61
|
+
|
62
|
+
context "created message" do
|
63
|
+
its(:id) { should eq build.message.id }
|
64
|
+
its(:name) { should eq build.message.name }
|
65
|
+
its(:src) { should eq build.message.src }
|
66
|
+
its(:sha) { should eq build.message.sha }
|
67
|
+
its(:deploy_key) { should eq build.message.deploy_key }
|
68
|
+
|
69
|
+
its(:job_id) { should eq job_id }
|
70
|
+
its(:before_script) { should be }
|
71
|
+
its(:script) { should be }
|
72
|
+
its(:matrix_keys) { should eq ['rvm:2.0.0'] }
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|