circleci-parallel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.rubocop.yml +35 -0
- data/.yardopts +2 -0
- data/Gemfile +11 -0
- data/LICENSE.txt +21 -0
- data/README.md +86 -0
- data/Rakefile +16 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/circle.yml +7 -0
- data/circleci-parallel.gemspec +24 -0
- data/lib/circleci/parallel.rb +178 -0
- data/lib/circleci/parallel/build.rb +32 -0
- data/lib/circleci/parallel/configuration.rb +88 -0
- data/lib/circleci/parallel/environment.rb +61 -0
- data/lib/circleci/parallel/hook.rb +23 -0
- data/lib/circleci/parallel/node.rb +48 -0
- data/lib/circleci/parallel/task/base.rb +32 -0
- data/lib/circleci/parallel/task/master.rb +61 -0
- data/lib/circleci/parallel/task/slave.rb +30 -0
- data/lib/circleci/parallel/version.rb +13 -0
- metadata +82 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a8f45e02994f05608bc9d43299f08a24fad83822
|
4
|
+
data.tar.gz: 1eacbf5a19c69c374ef56114053428f98ed83601
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b7b266accba3b7152ed128e07306bd04edf985819600951bf2c38437cd23d7d7e6e273d39850f86f0c3814247af35acfa83e8a3ac2876f8b975b74feb757166e
|
7
|
+
data.tar.gz: 2e8d863583c3d3195690f413945a0495d4ab1cbc57b758ca5a9f79b6a741b6267458c5b82e3cb4a70a14f7e1f9a8a1994079bdc9c273197fe2b3ba0ac9132d1f
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.1
|
3
|
+
|
4
|
+
Metrics/AbcSize:
|
5
|
+
Max: 16
|
6
|
+
|
7
|
+
Metrics/LineLength:
|
8
|
+
Max: 120
|
9
|
+
|
10
|
+
Style/AsciiComments:
|
11
|
+
Exclude:
|
12
|
+
- lib/circleci/parallel.rb
|
13
|
+
|
14
|
+
Style/ClassAndModuleChildren:
|
15
|
+
Exclude:
|
16
|
+
- spec/**/*
|
17
|
+
|
18
|
+
Style/Documentation:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/EmptyElse:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Style/GuardClause:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Style/IndentArray:
|
28
|
+
EnforcedStyle: consistent
|
29
|
+
|
30
|
+
Style/RegexpLiteral:
|
31
|
+
Exclude:
|
32
|
+
- '*.gemspec'
|
33
|
+
|
34
|
+
Style/SingleLineBlockParams:
|
35
|
+
Enabled: false
|
data/.yardopts
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Yuji Nakayama
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
# CircleCI::Parallel
|
2
|
+
|
3
|
+
**CircleCI::Parallel** provides simple APIs for joining [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
|
4
|
+
and sharing files between the nodes.
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
Add this line to your application's Gemfile:
|
9
|
+
|
10
|
+
```ruby
|
11
|
+
gem 'circleci-parallel'
|
12
|
+
```
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
```
|
17
|
+
$ bundle install
|
18
|
+
```
|
19
|
+
|
20
|
+
## Basic Usage
|
21
|
+
|
22
|
+
Before using CircleCI::Parallel:
|
23
|
+
|
24
|
+
* [Add `parallel: true`](https://circleci.com/docs/parallel-manual-setup/)
|
25
|
+
to the command that you'll use CircleCI::Parallel in your `circle.yml`.
|
26
|
+
* [Set up parallelism](https://circleci.com/docs/setting-up-parallelism/)
|
27
|
+
for your project from the CircleCI web console.
|
28
|
+
|
29
|
+
CircleCI::Parallel uses SSH for joining and transferring data between nodes.
|
30
|
+
|
31
|
+
```yaml
|
32
|
+
# circle.yml
|
33
|
+
test:
|
34
|
+
override:
|
35
|
+
- ruby test.rb:
|
36
|
+
parallel: true
|
37
|
+
```
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# test.rb
|
41
|
+
require 'circleci/parallel'
|
42
|
+
|
43
|
+
merged_data = {}
|
44
|
+
|
45
|
+
CircleCI::Parallel.configure do |config|
|
46
|
+
# This hook will be invoked on all the nodes.
|
47
|
+
# The current working directory in this hook is set to the local data directory
|
48
|
+
# where node specific data should be saved in.
|
49
|
+
config.before_join do
|
50
|
+
data = do_something
|
51
|
+
json = JSON.generate(data)
|
52
|
+
File.write('data.json', json)
|
53
|
+
end
|
54
|
+
|
55
|
+
# This hook will be invoked only on the master node after downloading all data from slave nodes.
|
56
|
+
# The current working directory in this hook is set to the download data directory
|
57
|
+
# where all node data are gathered into.
|
58
|
+
# The directory structure on the master node will be the following:
|
59
|
+
#
|
60
|
+
# .
|
61
|
+
# ├── node0
|
62
|
+
# │ └── node_specific_data_you_saved_on_node0.txt
|
63
|
+
# ├── node1
|
64
|
+
# │ └── node_specific_data_you_saved_on_node1.txt
|
65
|
+
# └── node2
|
66
|
+
# └── node_specific_data_you_saved_on_node2.txt
|
67
|
+
config.after_download do
|
68
|
+
Dir.glob('*/data.json') do |path|
|
69
|
+
json = File.read(path)
|
70
|
+
data = JSON.parse(json)
|
71
|
+
node_name = File.dirname(path)
|
72
|
+
merged_data[node_name] = data
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Join all nodes in the same build and gather all node data into the master node.
|
78
|
+
# Invoking this method blocks until the join and data downloads are complete.
|
79
|
+
CircleCI::Parallel.join
|
80
|
+
|
81
|
+
p merged_data
|
82
|
+
```
|
83
|
+
|
84
|
+
## License
|
85
|
+
|
86
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rspec/core/rake_task'
|
3
|
+
require 'rubocop/rake_task'
|
4
|
+
require 'yard'
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new do |task|
|
7
|
+
task.verbose = false
|
8
|
+
end
|
9
|
+
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
YARD::Rake::YardocTask.new
|
13
|
+
|
14
|
+
task default: [:spec, :rubocop]
|
15
|
+
|
16
|
+
task ci: [:spec, :rubocop]
|
data/bin/console
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'circleci/parallel'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
require 'pry'
|
10
|
+
Pry.start
|
data/bin/setup
ADDED
data/circle.yml
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'circleci/parallel/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'circleci-parallel'
|
8
|
+
spec.version = CircleCI::Parallel::Version.to_s
|
9
|
+
spec.authors = ['Yuji Nakayama']
|
10
|
+
spec.email = ['nkymyj@gmail.com']
|
11
|
+
|
12
|
+
spec.summary = "Provides Ruby APIs for joining CircleCI's parallel builds " \
|
13
|
+
'and sharing files between the builds'
|
14
|
+
spec.description = spec.summary
|
15
|
+
spec.homepage = 'https://github.com/increments/circleci-parallel'
|
16
|
+
spec.license = 'MIT'
|
17
|
+
|
18
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^spec/}) }
|
19
|
+
spec.bindir = 'exe'
|
20
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
+
spec.require_paths = ['lib']
|
22
|
+
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.12'
|
24
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'forwardable'
|
3
|
+
require 'circleci/parallel/environment'
|
4
|
+
|
5
|
+
module CircleCI
|
6
|
+
# Provides simple APIs for joining CircleCI's parallel builds and sharing files between the
|
7
|
+
# builds.
|
8
|
+
#
|
9
|
+
# @example
|
10
|
+
# merged_data = {}
|
11
|
+
#
|
12
|
+
# CircleCI::Parallel.configure do |config|
|
13
|
+
# config.before_join do
|
14
|
+
# data = do_something
|
15
|
+
# json = JSON.generate(data)
|
16
|
+
# File.write('data.json', json)
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# config.after_download do
|
20
|
+
# Dir.glob('*/data.json') do |path|
|
21
|
+
# json = File.read(path)
|
22
|
+
# data = JSON.parse(json)
|
23
|
+
# node_name = File.dirname(path)
|
24
|
+
# merged_data[node_name] = data
|
25
|
+
# end
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# CircleCI::Parallel.join
|
30
|
+
#
|
31
|
+
# p merged_data
|
32
|
+
module Parallel
|
33
|
+
extend SingleForwardable
|
34
|
+
|
35
|
+
# @api private
|
36
|
+
WORK_DIR = '/tmp/circleci-parallel'.freeze
|
37
|
+
|
38
|
+
# @api private
|
39
|
+
BASE_DATA_DIR = File.join(WORK_DIR, 'data')
|
40
|
+
|
41
|
+
# @api private
|
42
|
+
JOIN_MARKER_FILE = File.join(WORK_DIR, 'JOINING')
|
43
|
+
|
44
|
+
# @api private
|
45
|
+
DOWNLOAD_MARKER_FILE = File.join(WORK_DIR, 'DOWNLOADED')
|
46
|
+
|
47
|
+
# @!method configuration
|
48
|
+
|
49
|
+
# @!scope class
|
50
|
+
#
|
51
|
+
# Returns the current configuration.
|
52
|
+
#
|
53
|
+
# @return [Configuration] the current configuration
|
54
|
+
#
|
55
|
+
# @see .configure
|
56
|
+
def_delegator :environment, :configuration
|
57
|
+
|
58
|
+
# @!method configure
|
59
|
+
#
|
60
|
+
# @!scope class
|
61
|
+
#
|
62
|
+
# Provides a block for configuring RSpec::ComposableJSONMatchers.
|
63
|
+
#
|
64
|
+
# @yieldparam config [Configuration] the current configuration
|
65
|
+
#
|
66
|
+
# @return [void]
|
67
|
+
#
|
68
|
+
# @example
|
69
|
+
# CircleCI::Parallel.configure do |config|
|
70
|
+
# config.silent = true
|
71
|
+
# end
|
72
|
+
#
|
73
|
+
# @see .configuration
|
74
|
+
def_delegator :environment, :configure
|
75
|
+
|
76
|
+
# @!method current_build
|
77
|
+
#
|
78
|
+
# @!scope class
|
79
|
+
#
|
80
|
+
# Returns the current CircleCI build.
|
81
|
+
#
|
82
|
+
# @return [Build] the current build
|
83
|
+
#
|
84
|
+
# @see .current_node
|
85
|
+
def_delegator :environment, :current_build
|
86
|
+
|
87
|
+
# @!method current_node
|
88
|
+
#
|
89
|
+
# @!scope class
|
90
|
+
#
|
91
|
+
# Returns the current CircleCI node.
|
92
|
+
#
|
93
|
+
# @return [Build] the current node
|
94
|
+
#
|
95
|
+
# @see .current_build
|
96
|
+
def_delegator :environment, :current_node
|
97
|
+
|
98
|
+
# @!method join
|
99
|
+
#
|
100
|
+
# @!scope class
|
101
|
+
#
|
102
|
+
# Join all nodes in the same build and gather all node data into the master node.
|
103
|
+
# Invoking this method blocks until the join and data downloads are complete.
|
104
|
+
#
|
105
|
+
# @raise [RuntimeError] when `CIRCLECI` environment variable is not set
|
106
|
+
#
|
107
|
+
# @see CircleCI::Parallel::Configuration#before_join
|
108
|
+
# @see CircleCI::Parallel::Configuration#after_join
|
109
|
+
# @see CircleCI::Parallel::Configuration#after_download
|
110
|
+
def_delegator :environment, :join
|
111
|
+
|
112
|
+
# @api private
|
113
|
+
# @!method puts
|
114
|
+
# @!scope class
|
115
|
+
def_delegator :environment, :puts
|
116
|
+
|
117
|
+
class << self
|
118
|
+
# Returns the local data directory where node specific data should be saved in.
|
119
|
+
#
|
120
|
+
# @return [String] the local data directory
|
121
|
+
#
|
122
|
+
# @example
|
123
|
+
# path = File.join(CircleCI::Parallel.local_data_dir, 'data.json')
|
124
|
+
# File.write(path, JSON.generate(some_data))
|
125
|
+
#
|
126
|
+
# @see CircleCI::Parallel::Configuration#before_join
|
127
|
+
# @see CircleCI::Parallel::Configuration#after_join
|
128
|
+
def local_data_dir
|
129
|
+
current_node.data_dir.tap do |path|
|
130
|
+
FileUtils.makedirs(path) unless Dir.exist?(path)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
# Returns the download data directory where all node data will be downloaded.
|
135
|
+
# Note that only master node downloads data from other slave node.
|
136
|
+
# When the downloads are complete, the directory structure on the master node will be the
|
137
|
+
# following:
|
138
|
+
#
|
139
|
+
# .
|
140
|
+
# ├── node0
|
141
|
+
# │ └── node_specific_data_you_saved_on_node0.txt
|
142
|
+
# ├── node1
|
143
|
+
# │ └── node_specific_data_you_saved_on_node1.txt
|
144
|
+
# └── node2
|
145
|
+
# └── node_specific_data_you_saved_on_node2.txt
|
146
|
+
#
|
147
|
+
# @return [String] the download data directory
|
148
|
+
#
|
149
|
+
# @example
|
150
|
+
# Dir.chdir(CircleCI::Parallel.download_data_dir) do
|
151
|
+
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
152
|
+
# data = JSON.parse(File.read(path))
|
153
|
+
# node_name = File.dirname(path)
|
154
|
+
# merged_data[node_name] = data
|
155
|
+
# end
|
156
|
+
# end
|
157
|
+
#
|
158
|
+
# @see CircleCI::Parallel::Configuration#after_download
|
159
|
+
def download_data_dir
|
160
|
+
BASE_DATA_DIR.tap do |path|
|
161
|
+
FileUtils.makedirs(path) unless Dir.exist?(path)
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
# @api private
|
166
|
+
def reset!
|
167
|
+
environment.clean
|
168
|
+
@environment = nil
|
169
|
+
end
|
170
|
+
|
171
|
+
private
|
172
|
+
|
173
|
+
def environment
|
174
|
+
@environment ||= Environment.new
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'circleci/parallel/node'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
# Represents a CircleCI build.
|
6
|
+
class Build
|
7
|
+
attr_reader :number, :node_count
|
8
|
+
|
9
|
+
# @param number [Integer] the build number (`CIRCLE_BUILD_NUM`)
|
10
|
+
# @param node_count [Integer] node count of the build (`CIRCLE_NODE_TOTAL`)
|
11
|
+
def initialize(number, node_count)
|
12
|
+
@number = number
|
13
|
+
@node_count = node_count
|
14
|
+
end
|
15
|
+
|
16
|
+
def ==(other)
|
17
|
+
number == other.number
|
18
|
+
end
|
19
|
+
|
20
|
+
alias eql? ==
|
21
|
+
|
22
|
+
def hash
|
23
|
+
number.hash ^ node_count.hash
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [Array<Node>] nodes of the build
|
27
|
+
def nodes
|
28
|
+
@nodes ||= Array.new(node_count) { |index| Node.new(self, index) }.freeze
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
# @return [Boolean] whether progress messages should be outputted to STDOUT (default: false)
|
7
|
+
attr_accessor :silent
|
8
|
+
|
9
|
+
# @api private
|
10
|
+
attr_reader :before_join_hook, :after_join_hook, :after_download_hook
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@silent = false
|
14
|
+
@before_join_hook = @after_join_hook = @after_download_hook = Hook.new
|
15
|
+
end
|
16
|
+
|
17
|
+
# Defines a callback that will be invoked on all nodes before joining nodes.
|
18
|
+
#
|
19
|
+
# @param chdir [Boolean] whether the callback should be invoked while chaging the current
|
20
|
+
# working directory to the local data directory.
|
21
|
+
#
|
22
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
23
|
+
#
|
24
|
+
# @return [void]
|
25
|
+
#
|
26
|
+
# @example
|
27
|
+
# CircleCI::Parallel.configure do |config|
|
28
|
+
# config.before_join do
|
29
|
+
# File.write('data.json', JSON.generate(some_data))
|
30
|
+
# end
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# @see CircleCI::Parallel.local_data_dir
|
34
|
+
def before_join(chdir: true, &block)
|
35
|
+
@before_join_hook = Hook.new(block, chdir)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Defines a callback that will be invoked on all nodes after joining nodes.
|
39
|
+
#
|
40
|
+
# @param chdir [Boolean] whether the callback should be invoked while chaging the current
|
41
|
+
# working directory to the local data directory.
|
42
|
+
#
|
43
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
44
|
+
#
|
45
|
+
# @return [void]
|
46
|
+
#
|
47
|
+
# @example
|
48
|
+
# CircleCI::Parallel.configure do |config|
|
49
|
+
# config.after_join do
|
50
|
+
# clean_some_intermediate_data
|
51
|
+
# end
|
52
|
+
# end
|
53
|
+
#
|
54
|
+
# @see CircleCI::Parallel.local_data_dir
|
55
|
+
def after_join(chdir: true, &block)
|
56
|
+
@after_join_hook = Hook.new(block, chdir)
|
57
|
+
end
|
58
|
+
|
59
|
+
# Defines a callback that will be invoked only on the master node after downloading all data
|
60
|
+
# from slave nodes.
|
61
|
+
#
|
62
|
+
# @param chdir [Boolean] whether the callback should be invoked while chaging the current
|
63
|
+
# working directory to the download data directory.
|
64
|
+
#
|
65
|
+
# @yieldparam download_data_dir [String] the path to the download data directory
|
66
|
+
#
|
67
|
+
# @return [void]
|
68
|
+
#
|
69
|
+
# @example
|
70
|
+
# CircleCI::Parallel.configure do |config|
|
71
|
+
# config.after_download do
|
72
|
+
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
73
|
+
# data = JSON.parse(File.read(path))
|
74
|
+
# node_name = File.dirname(path)
|
75
|
+
# merged_data[node_name] = data
|
76
|
+
# end
|
77
|
+
#
|
78
|
+
# File.write('merged_data.json', JSON.generate(merged_data))
|
79
|
+
# end
|
80
|
+
# end
|
81
|
+
#
|
82
|
+
# @see CircleCI::Parallel.download_data_dir
|
83
|
+
def after_download(chdir: true, &block)
|
84
|
+
@after_download_hook = Hook.new(block, chdir)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'circleci/parallel/build'
|
3
|
+
require 'circleci/parallel/configuration'
|
4
|
+
require 'circleci/parallel/node'
|
5
|
+
require 'circleci/parallel/task/master'
|
6
|
+
require 'circleci/parallel/task/slave'
|
7
|
+
|
8
|
+
module CircleCI
|
9
|
+
module Parallel
|
10
|
+
# @api private
|
11
|
+
class Environment
|
12
|
+
def configuration
|
13
|
+
@configuration ||= Configuration.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def configure
|
17
|
+
yield configuration
|
18
|
+
end
|
19
|
+
|
20
|
+
def current_build
|
21
|
+
@current_build ||= Build.new(ENV['CIRCLE_BUILD_NUM'].to_i, ENV['CIRCLE_NODE_TOTAL'].to_i)
|
22
|
+
end
|
23
|
+
|
24
|
+
def current_node
|
25
|
+
@current_node ||= Node.new(current_build, ENV['CIRCLE_NODE_INDEX'].to_i)
|
26
|
+
end
|
27
|
+
|
28
|
+
def join
|
29
|
+
validate!
|
30
|
+
task.run
|
31
|
+
end
|
32
|
+
|
33
|
+
def puts(*args)
|
34
|
+
Kernel.puts(*args) unless configuration.silent
|
35
|
+
end
|
36
|
+
|
37
|
+
def clean
|
38
|
+
FileUtils.rmtree(WORK_DIR) if Dir.exist?(WORK_DIR)
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def validate!
|
44
|
+
raise 'The current environment is not on CircleCI.' unless ENV['CIRCLECI']
|
45
|
+
|
46
|
+
unless ENV['CIRCLE_NODE_TOTAL']
|
47
|
+
warn 'Environment variable CIRCLE_NODE_TOTAL is not set. ' \
|
48
|
+
'Maybe you forgot adding `parallel: true` to your circle.yml? ' \
|
49
|
+
'https://circleci.com/docs/parallel-manual-setup/'
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def task
|
54
|
+
@task ||= begin
|
55
|
+
task_class = current_node.master? ? Task::Master : Task::Slave
|
56
|
+
task_class.new(current_node, configuration)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module CircleCI
|
2
|
+
module Parallel
|
3
|
+
# @api private
|
4
|
+
class Hook
|
5
|
+
attr_reader :proc, :chdir
|
6
|
+
|
7
|
+
def initialize(proc = nil, chdir = true)
|
8
|
+
@proc = proc
|
9
|
+
@chdir = chdir
|
10
|
+
end
|
11
|
+
|
12
|
+
def call(dir)
|
13
|
+
return unless proc
|
14
|
+
|
15
|
+
if chdir
|
16
|
+
Dir.chdir(dir, &proc)
|
17
|
+
else
|
18
|
+
proc.call(dir)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module CircleCI
|
2
|
+
module Parallel
|
3
|
+
# Represents a CircleCI node.
|
4
|
+
class Node
|
5
|
+
attr_reader :build, :index
|
6
|
+
|
7
|
+
# @param build [Build] the build that the node belongs to
|
8
|
+
# @param index [Integer] node index (`CIRCLE_NODE_INDEX`)
|
9
|
+
def initialize(build, index)
|
10
|
+
@build = build
|
11
|
+
@index = index
|
12
|
+
end
|
13
|
+
|
14
|
+
def ==(other)
|
15
|
+
build == other.build && index == other.index
|
16
|
+
end
|
17
|
+
|
18
|
+
alias eql? ==
|
19
|
+
|
20
|
+
def hash
|
21
|
+
build.hash ^ index.hash
|
22
|
+
end
|
23
|
+
|
24
|
+
# @return [Boolean] whether the node is the master node or not
|
25
|
+
def master?
|
26
|
+
index.zero?
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [String] the hostname that can be used for `ssh` command to connect between nodes
|
30
|
+
def ssh_host
|
31
|
+
# https://circleci.com/docs/ssh-between-build-containers/
|
32
|
+
"node#{index}"
|
33
|
+
end
|
34
|
+
|
35
|
+
# @return [String] the local data directory where node specific data should be saved in
|
36
|
+
#
|
37
|
+
# @see CircleCI::Parallel.local_data_dir
|
38
|
+
def data_dir
|
39
|
+
File.join(BASE_DATA_DIR, ssh_host)
|
40
|
+
end
|
41
|
+
|
42
|
+
# @return [Array<Node>] other nodes of the same build
|
43
|
+
def other_nodes
|
44
|
+
@other_nodes ||= (build.nodes - [self]).freeze
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
module Task
|
6
|
+
# @api private
|
7
|
+
class Base
|
8
|
+
attr_reader :node, :configuration
|
9
|
+
|
10
|
+
def initialize(node, configuration)
|
11
|
+
@node = node
|
12
|
+
@configuration = configuration
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def create_node_data_dir
|
22
|
+
FileUtils.makedirs(node.data_dir)
|
23
|
+
end
|
24
|
+
|
25
|
+
def mark_as_joining
|
26
|
+
Parallel.puts('Joining CircleCI nodes...')
|
27
|
+
File.write(JOIN_MARKER_FILE, '')
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'circleci/parallel/task/base'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
module Task
|
6
|
+
# @api private
|
7
|
+
class Master < Base
|
8
|
+
def run
|
9
|
+
create_node_data_dir
|
10
|
+
configuration.before_join_hook.call(node.data_dir)
|
11
|
+
mark_as_joining
|
12
|
+
download_from_slave_nodes
|
13
|
+
configuration.after_download_hook.call(BASE_DATA_DIR)
|
14
|
+
configuration.after_join_hook.call(node.data_dir)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def download_from_slave_nodes
|
20
|
+
# TODO: Consider implementing timeout mechanism
|
21
|
+
Parallel.puts('Waiting for slave nodes to be ready for download...')
|
22
|
+
loop do
|
23
|
+
downloaders.each(&:download)
|
24
|
+
break if downloaders.all?(&:downloaded?)
|
25
|
+
Kernel.sleep(1)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def downloaders
|
30
|
+
@downloaders ||= node.other_nodes.map { |other_node| Downloader.new(other_node) }
|
31
|
+
end
|
32
|
+
|
33
|
+
Downloader = Struct.new(:node) do
|
34
|
+
def ready_for_download?
|
35
|
+
Kernel.system('ssh', node.ssh_host, 'test', '-f', JOIN_MARKER_FILE)
|
36
|
+
end
|
37
|
+
|
38
|
+
def download
|
39
|
+
return if downloaded?
|
40
|
+
return unless ready_for_download?
|
41
|
+
Parallel.puts("Downloading data from #{node.ssh_host}...")
|
42
|
+
@downloaded = scp
|
43
|
+
mark_as_downloaded if downloaded?
|
44
|
+
end
|
45
|
+
|
46
|
+
def scp
|
47
|
+
Kernel.system('scp', '-q', '-r', "#{node.ssh_host}:#{node.data_dir}", BASE_DATA_DIR)
|
48
|
+
end
|
49
|
+
|
50
|
+
def downloaded?
|
51
|
+
@downloaded
|
52
|
+
end
|
53
|
+
|
54
|
+
def mark_as_downloaded
|
55
|
+
Kernel.system('ssh', node.ssh_host, 'touch', DOWNLOAD_MARKER_FILE)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'circleci/parallel/task/base'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
module Task
|
6
|
+
# @api private
|
7
|
+
class Slave < Base
|
8
|
+
def run
|
9
|
+
create_node_data_dir
|
10
|
+
configuration.before_join_hook.call(node.data_dir)
|
11
|
+
mark_as_joining
|
12
|
+
wait_for_master_node_to_download
|
13
|
+
configuration.after_join_hook.call(node.data_dir)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def wait_for_master_node_to_download
|
19
|
+
# TODO: Consider implementing timeout mechanism
|
20
|
+
Parallel.puts('Waiting for master node to download data...')
|
21
|
+
Kernel.sleep(1) until downloaded?
|
22
|
+
end
|
23
|
+
|
24
|
+
def downloaded?
|
25
|
+
File.exist?(DOWNLOAD_MARKER_FILE)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: circleci-parallel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yuji Nakayama
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-08-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.12'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.12'
|
27
|
+
description: Provides Ruby APIs for joining CircleCI's parallel builds and sharing
|
28
|
+
files between the builds
|
29
|
+
email:
|
30
|
+
- nkymyj@gmail.com
|
31
|
+
executables: []
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- ".gitignore"
|
36
|
+
- ".rspec"
|
37
|
+
- ".rubocop.yml"
|
38
|
+
- ".yardopts"
|
39
|
+
- Gemfile
|
40
|
+
- LICENSE.txt
|
41
|
+
- README.md
|
42
|
+
- Rakefile
|
43
|
+
- bin/console
|
44
|
+
- bin/setup
|
45
|
+
- circle.yml
|
46
|
+
- circleci-parallel.gemspec
|
47
|
+
- lib/circleci/parallel.rb
|
48
|
+
- lib/circleci/parallel/build.rb
|
49
|
+
- lib/circleci/parallel/configuration.rb
|
50
|
+
- lib/circleci/parallel/environment.rb
|
51
|
+
- lib/circleci/parallel/hook.rb
|
52
|
+
- lib/circleci/parallel/node.rb
|
53
|
+
- lib/circleci/parallel/task/base.rb
|
54
|
+
- lib/circleci/parallel/task/master.rb
|
55
|
+
- lib/circleci/parallel/task/slave.rb
|
56
|
+
- lib/circleci/parallel/version.rb
|
57
|
+
homepage: https://github.com/increments/circleci-parallel
|
58
|
+
licenses:
|
59
|
+
- MIT
|
60
|
+
metadata: {}
|
61
|
+
post_install_message:
|
62
|
+
rdoc_options: []
|
63
|
+
require_paths:
|
64
|
+
- lib
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
requirements: []
|
76
|
+
rubyforge_project:
|
77
|
+
rubygems_version: 2.6.6
|
78
|
+
signing_key:
|
79
|
+
specification_version: 4
|
80
|
+
summary: Provides Ruby APIs for joining CircleCI's parallel builds and sharing files
|
81
|
+
between the builds
|
82
|
+
test_files: []
|