circleci-parallel 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -0
- data/CHANGELOG.md +15 -1
- data/README.md +6 -6
- data/Rakefile +2 -2
- data/circleci-parallel.gemspec +3 -2
- data/lib/circleci/parallel.rb +29 -18
- data/lib/circleci/parallel/configuration.rb +36 -66
- data/lib/circleci/parallel/configuration/configuration_collection_proxy.rb +64 -0
- data/lib/circleci/parallel/configuration/master_node_configuration.rb +102 -0
- data/lib/circleci/parallel/configuration/slave_node_configuration.rb +59 -0
- data/lib/circleci/parallel/environment.rb +10 -8
- data/lib/circleci/parallel/task/base.rb +2 -2
- data/lib/circleci/parallel/task/master.rb +6 -5
- data/lib/circleci/parallel/task/slave.rb +3 -3
- data/lib/circleci/parallel/version.rb +2 -2
- metadata +10 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc704707586c01c1552e51c7f025494025a3719f
|
4
|
+
data.tar.gz: 0edc41e83e78e85f9925b1aa0a640fb832372800
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad1a77fa741b4e494c1eef4c4af017810546748aaee18d987fbb00c5225924adf334a6e74829260a78e3c7a05eb8b65e591b2af8bb735086c09106c58861fb82
|
7
|
+
data.tar.gz: f356cb223fc9ba5b9f5ab1e7220b834945d6798304a6e678e8a8b1e0b926d2bb6fc06e0491f6bae2ab5cdb95542401a60ea603ab8fcaaf18a66a801802ee6623
|
data/.rubocop.yml
CHANGED
@@ -1,9 +1,21 @@
|
|
1
1
|
AllCops:
|
2
2
|
TargetRubyVersion: 2.1
|
3
3
|
|
4
|
+
Layout/MultilineMethodCallIndentation:
|
5
|
+
Exclude:
|
6
|
+
- spec/**/*
|
7
|
+
|
8
|
+
Lint/AmbiguousBlockAssociation:
|
9
|
+
Exclude:
|
10
|
+
- spec/**/*
|
11
|
+
|
4
12
|
Metrics/AbcSize:
|
5
13
|
Max: 17
|
6
14
|
|
15
|
+
Metrics/BlockLength:
|
16
|
+
Exclude:
|
17
|
+
- spec/**/*
|
18
|
+
|
7
19
|
Metrics/LineLength:
|
8
20
|
Max: 120
|
9
21
|
|
data/CHANGELOG.md
CHANGED
@@ -2,13 +2,27 @@
|
|
2
2
|
|
3
3
|
## Development
|
4
4
|
|
5
|
+
## v0.5.0
|
6
|
+
|
7
|
+
* Add improved hook configuration APIs.
|
8
|
+
* `config.on_every_node.before_sync`
|
9
|
+
* `config.on_every_node.after_sync`
|
10
|
+
* `config.on_master_node.before_sync`
|
11
|
+
* `config.on_master_node.after_sync`
|
12
|
+
* `config.on_master_node.before_download`
|
13
|
+
* `config.on_master_node.after_download`
|
14
|
+
* `config.on_each_slave_node.before_sync`
|
15
|
+
* `config.on_each_slave_node.after_sync`
|
16
|
+
* Deprecate `#before_join`, `#after_join`, and `#after_download` in `Configuration`.
|
17
|
+
* Add `Parallel.sync` and deprecate `Parallel.join`.
|
18
|
+
|
5
19
|
## v0.4.1
|
6
20
|
|
7
21
|
* Tweak UI messsages.
|
8
22
|
|
9
23
|
## v0.4.0
|
10
24
|
|
11
|
-
* Print "Done." when
|
25
|
+
* Print "Done." when sync is complete.
|
12
26
|
|
13
27
|
## v0.3.0
|
14
28
|
|
data/README.md
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
# CircleCI::Parallel
|
8
8
|
|
9
|
-
**CircleCI::Parallel** provides simple APIs for syncing [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
|
9
|
+
**CircleCI::Parallel** provides simple Ruby APIs for syncing [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
|
10
10
|
and transferring files between the nodes.
|
11
11
|
|
12
12
|
## Installation
|
@@ -47,10 +47,10 @@ test:
|
|
47
47
|
require 'circleci/parallel'
|
48
48
|
|
49
49
|
CircleCI::Parallel.configure do |config|
|
50
|
-
# This hook will be invoked on
|
50
|
+
# This hook will be invoked on every node.
|
51
51
|
# The current working directory in this hook is set to the local data directory
|
52
52
|
# where node specific data should be saved in.
|
53
|
-
config.
|
53
|
+
config.on_every_node.before_sync do
|
54
54
|
data = do_something
|
55
55
|
json = JSON.generate(data)
|
56
56
|
File.write('data.json', json)
|
@@ -68,7 +68,7 @@ CircleCI::Parallel.configure do |config|
|
|
68
68
|
# │ └── node_specific_data_you_saved_on_node1.txt
|
69
69
|
# └── node2
|
70
70
|
# └── node_specific_data_you_saved_on_node2.txt
|
71
|
-
config.after_download do
|
71
|
+
config.on_master_node.after_download do
|
72
72
|
merged_data = {}
|
73
73
|
|
74
74
|
Dir.glob('*/data.json') do |path|
|
@@ -83,8 +83,8 @@ CircleCI::Parallel.configure do |config|
|
|
83
83
|
end
|
84
84
|
|
85
85
|
# Sync all nodes in the same build and gather all node data into the master node.
|
86
|
-
# Invoking this method blocks until the
|
87
|
-
CircleCI::Parallel.
|
86
|
+
# Invoking this method blocks until the sync and the data transfer is complete.
|
87
|
+
CircleCI::Parallel.sync
|
88
88
|
```
|
89
89
|
|
90
90
|
See [API docs](http://www.rubydoc.info/gems/circleci-parallel) for more details.
|
data/Rakefile
CHANGED
data/circleci-parallel.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'circleci/parallel/version'
|
@@ -9,8 +10,8 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ['Yuji Nakayama']
|
10
11
|
spec.email = ['nkymyj@gmail.com']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
'and
|
13
|
+
spec.summary = 'Provides Ruby APIs for syncing CircleCI parallel nodes ' \
|
14
|
+
'and transferring files between the nodes'
|
14
15
|
spec.description = spec.summary
|
15
16
|
spec.homepage = 'https://github.com/increments/circleci-parallel'
|
16
17
|
spec.license = 'MIT'
|
data/lib/circleci/parallel.rb
CHANGED
@@ -3,20 +3,20 @@ require 'forwardable'
|
|
3
3
|
require 'circleci/parallel/environment'
|
4
4
|
|
5
5
|
module CircleCI
|
6
|
-
# Provides simple APIs for
|
7
|
-
#
|
6
|
+
# Provides simple APIs for syncing CircleCI parallel nodes and transferring files between the
|
7
|
+
# nodes.
|
8
8
|
#
|
9
9
|
# @example
|
10
10
|
# merged_data = {}
|
11
11
|
#
|
12
12
|
# CircleCI::Parallel.configure do |config|
|
13
|
-
# config.
|
13
|
+
# config.on_every_node.before_sync do
|
14
14
|
# data = do_something
|
15
15
|
# json = JSON.generate(data)
|
16
16
|
# File.write('data.json', json)
|
17
17
|
# end
|
18
18
|
#
|
19
|
-
# config.after_download do
|
19
|
+
# config.on_master_node.after_download do
|
20
20
|
# Dir.glob('*/data.json') do |path|
|
21
21
|
# json = File.read(path)
|
22
22
|
# data = JSON.parse(json)
|
@@ -26,7 +26,7 @@ module CircleCI
|
|
26
26
|
# end
|
27
27
|
# end
|
28
28
|
#
|
29
|
-
# CircleCI::Parallel.
|
29
|
+
# CircleCI::Parallel.sync
|
30
30
|
#
|
31
31
|
# p merged_data
|
32
32
|
module Parallel
|
@@ -39,7 +39,7 @@ module CircleCI
|
|
39
39
|
BASE_DATA_DIR = File.join(WORK_DIR, 'data')
|
40
40
|
|
41
41
|
# @api private
|
42
|
-
|
42
|
+
SYNC_MARKER_FILE = File.join(WORK_DIR, 'SYNCING')
|
43
43
|
|
44
44
|
# @api private
|
45
45
|
DOWNLOAD_MARKER_FILE = File.join(WORK_DIR, 'DOWNLOADED')
|
@@ -59,7 +59,7 @@ module CircleCI
|
|
59
59
|
#
|
60
60
|
# @!scope class
|
61
61
|
#
|
62
|
-
# Provides a block for configuring
|
62
|
+
# Provides a block for configuring CircleCI::Parallel.
|
63
63
|
#
|
64
64
|
# @yieldparam config [Configuration] the current configuration
|
65
65
|
#
|
@@ -90,24 +90,27 @@ module CircleCI
|
|
90
90
|
#
|
91
91
|
# Returns the current CircleCI node.
|
92
92
|
#
|
93
|
-
# @return [
|
93
|
+
# @return [Node] the current node
|
94
94
|
#
|
95
95
|
# @see .current_build
|
96
96
|
def_delegator :environment, :current_node
|
97
97
|
|
98
|
-
# @!method
|
98
|
+
# @!method sync
|
99
99
|
#
|
100
100
|
# @!scope class
|
101
101
|
#
|
102
|
-
#
|
103
|
-
# Invoking this method blocks until the
|
102
|
+
# Sync all nodes in the same build and gather all node data into the master node.
|
103
|
+
# Invoking this method blocks until the sync and data downloads are complete.
|
104
104
|
#
|
105
105
|
# @raise [RuntimeError] when `CIRCLECI` environment variable is not set
|
106
106
|
#
|
107
|
-
# @see CircleCI::Parallel::
|
108
|
-
# @see CircleCI::Parallel::
|
109
|
-
# @see CircleCI::Parallel::
|
110
|
-
|
107
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_sync
|
108
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_download
|
109
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_download
|
110
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_sync
|
111
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#before_sync
|
112
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#after_sync
|
113
|
+
def_delegator :environment, :sync
|
111
114
|
|
112
115
|
# @api private
|
113
116
|
# @!method puts
|
@@ -123,8 +126,10 @@ module CircleCI
|
|
123
126
|
# path = File.join(CircleCI::Parallel.local_data_dir, 'data.json')
|
124
127
|
# File.write(path, JSON.generate(some_data))
|
125
128
|
#
|
126
|
-
# @see CircleCI::Parallel::
|
127
|
-
# @see CircleCI::Parallel::
|
129
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_sync
|
130
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_sync
|
131
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#before_sync
|
132
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#after_sync
|
128
133
|
def local_data_dir
|
129
134
|
current_node.data_dir.tap do |path|
|
130
135
|
FileUtils.makedirs(path) unless Dir.exist?(path)
|
@@ -155,7 +160,8 @@ module CircleCI
|
|
155
160
|
# end
|
156
161
|
# end
|
157
162
|
#
|
158
|
-
# @see CircleCI::Parallel::
|
163
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_download
|
164
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_download
|
159
165
|
def download_data_dir
|
160
166
|
BASE_DATA_DIR.tap do |path|
|
161
167
|
FileUtils.makedirs(path) unless Dir.exist?(path)
|
@@ -168,6 +174,11 @@ module CircleCI
|
|
168
174
|
@environment = nil
|
169
175
|
end
|
170
176
|
|
177
|
+
# @deprecated Use `.sync` instead.
|
178
|
+
def join
|
179
|
+
sync
|
180
|
+
end
|
181
|
+
|
171
182
|
private
|
172
183
|
|
173
184
|
def environment
|
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'circleci/parallel/configuration/configuration_collection_proxy'
|
2
|
+
require 'circleci/parallel/configuration/master_node_configuration'
|
3
|
+
require 'circleci/parallel/configuration/slave_node_configuration'
|
1
4
|
require 'circleci/parallel/hook'
|
2
5
|
|
3
6
|
module CircleCI
|
@@ -9,83 +12,50 @@ module CircleCI
|
|
9
12
|
# @return [Boolean] whether mock mode is enabled (default: false)
|
10
13
|
attr_accessor :mock_mode
|
11
14
|
|
12
|
-
# @api private
|
13
|
-
attr_reader :before_join_hook, :after_join_hook, :after_download_hook
|
14
|
-
|
15
15
|
def initialize
|
16
16
|
@silent = false
|
17
17
|
@mock_mode = false
|
18
|
-
@before_join_hook = @after_join_hook = @after_download_hook = Hook.new
|
19
18
|
end
|
20
19
|
|
21
|
-
#
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
#
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
# @
|
20
|
+
# @return [ConfigurationCollectionProxy]
|
21
|
+
def on_every_node
|
22
|
+
@every_node_configuration ||=
|
23
|
+
ConfigurationCollectionProxy.new(master_node_configuration, slave_node_configuration)
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [MasterNodeConfiguration]
|
27
|
+
def on_master_node
|
28
|
+
master_node_configuration
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [SlaveNodeConfiguration]
|
32
|
+
def on_each_slave_node
|
33
|
+
slave_node_configuration
|
34
|
+
end
|
35
|
+
|
36
|
+
# @deprecated Use `on_every_node.before_sync` instead.
|
38
37
|
def before_join(chdir: true, &block)
|
39
|
-
|
38
|
+
on_every_node.before_sync(chdir: chdir, &block)
|
40
39
|
end
|
41
40
|
|
42
|
-
#
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
# @
|
48
|
-
#
|
49
|
-
# @return [void]
|
50
|
-
#
|
51
|
-
# @example
|
52
|
-
# CircleCI::Parallel.configure do |config|
|
53
|
-
# config.after_join do
|
54
|
-
# clean_some_intermediate_data
|
55
|
-
# end
|
56
|
-
# end
|
57
|
-
#
|
58
|
-
# @see CircleCI::Parallel.local_data_dir
|
41
|
+
# @deprecated Use `on_master_node.after_download` instead.
|
42
|
+
def after_download(chdir: true, &block)
|
43
|
+
on_master_node.after_download(chdir: chdir, &block)
|
44
|
+
end
|
45
|
+
|
46
|
+
# @deprecated Use `on_every_node.after_sync` instead.
|
59
47
|
def after_join(chdir: true, &block)
|
60
|
-
|
48
|
+
on_every_node.after_sync(chdir: chdir, &block)
|
61
49
|
end
|
62
50
|
|
63
|
-
#
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
#
|
69
|
-
|
70
|
-
|
71
|
-
# @return [void]
|
72
|
-
#
|
73
|
-
# @example
|
74
|
-
# CircleCI::Parallel.configure do |config|
|
75
|
-
# config.after_download do
|
76
|
-
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
77
|
-
# data = JSON.parse(File.read(path))
|
78
|
-
# node_name = File.dirname(path)
|
79
|
-
# merged_data[node_name] = data
|
80
|
-
# end
|
81
|
-
#
|
82
|
-
# File.write('merged_data.json', JSON.generate(merged_data))
|
83
|
-
# end
|
84
|
-
# end
|
85
|
-
#
|
86
|
-
# @see CircleCI::Parallel.download_data_dir
|
87
|
-
def after_download(chdir: true, &block)
|
88
|
-
@after_download_hook = Hook.new(block, chdir)
|
51
|
+
# @api private
|
52
|
+
def master_node_configuration
|
53
|
+
@master_node_configuration ||= MasterNodeConfiguration.new
|
54
|
+
end
|
55
|
+
|
56
|
+
# @api private
|
57
|
+
def slave_node_configuration
|
58
|
+
@slave_node_configuration ||= SlaveNodeConfiguration.new
|
89
59
|
end
|
90
60
|
end
|
91
61
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
# A convenient proxy to modify both master node and slave node configuration at once.
|
7
|
+
class ConfigurationCollectionProxy
|
8
|
+
# @api private
|
9
|
+
attr_reader :configurations
|
10
|
+
|
11
|
+
# @api private
|
12
|
+
def initialize(*configurations)
|
13
|
+
@configurations = configurations
|
14
|
+
end
|
15
|
+
|
16
|
+
# Defines a callback that will be invoked on every node before syncing all nodes.
|
17
|
+
#
|
18
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
19
|
+
# working directory to the local data directory.
|
20
|
+
#
|
21
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
22
|
+
#
|
23
|
+
# @return [void]
|
24
|
+
#
|
25
|
+
# @example
|
26
|
+
# CircleCI::Parallel.configure do |config|
|
27
|
+
# config.on_every_node.before_sync do
|
28
|
+
# File.write('data.json', JSON.generate(some_data))
|
29
|
+
# end
|
30
|
+
# end
|
31
|
+
#
|
32
|
+
# @see CircleCI::Parallel.local_data_dir
|
33
|
+
def before_sync(chdir: true, &block)
|
34
|
+
configurations.each do |configuration|
|
35
|
+
configuration.before_sync(chdir: chdir, &block)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Defines a callback that will be invoked on every node after syncing all nodes.
|
40
|
+
#
|
41
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
42
|
+
# working directory to the local data directory.
|
43
|
+
#
|
44
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
45
|
+
#
|
46
|
+
# @return [void]
|
47
|
+
#
|
48
|
+
# @example
|
49
|
+
# CircleCI::Parallel.configure do |config|
|
50
|
+
# config.on_each_slave_node.after_sync do
|
51
|
+
# clean_some_intermediate_data
|
52
|
+
# end
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# @see CircleCI::Parallel.local_data_dir
|
56
|
+
def after_sync(chdir: true, &block)
|
57
|
+
configurations.each do |configuration|
|
58
|
+
configuration.after_sync(chdir: chdir, &block)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
class MasterNodeConfiguration
|
7
|
+
# @api private
|
8
|
+
attr_reader :before_sync_hook, :before_download_hook, :after_download_hook, :after_sync_hook
|
9
|
+
|
10
|
+
# @api private
|
11
|
+
def initialize
|
12
|
+
@before_sync_hook = @before_download_hook = @after_download_hook = @after_sync_hook = Hook.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# Defines a callback that will be invoked on the master node before syncing all nodes.
|
16
|
+
#
|
17
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
18
|
+
# working directory to the local data directory.
|
19
|
+
#
|
20
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
21
|
+
#
|
22
|
+
# @return [void]
|
23
|
+
#
|
24
|
+
# @example
|
25
|
+
# CircleCI::Parallel.configure do |config|
|
26
|
+
# config.on_master_node.before_sync do
|
27
|
+
# File.write('data.json', JSON.generate(some_data))
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# @see CircleCI::Parallel.local_data_dir
|
32
|
+
def before_sync(chdir: true, &block)
|
33
|
+
@before_sync_hook = Hook.new(block, chdir)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Defines a callback that will be invoked on the master node before downloading all data
|
37
|
+
# from slave nodes.
|
38
|
+
#
|
39
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
40
|
+
# working directory to the download data directory.
|
41
|
+
#
|
42
|
+
# @yieldparam download_data_dir [String] the path to the download data directory
|
43
|
+
#
|
44
|
+
# @return [void]
|
45
|
+
#
|
46
|
+
# @see CircleCI::Parallel.download_data_dir
|
47
|
+
def before_download(chdir: true, &block)
|
48
|
+
@before_download_hook = Hook.new(block, chdir)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Defines a callback that will be invoked on the master node after downloading all data
|
52
|
+
# from slave nodes.
|
53
|
+
#
|
54
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
55
|
+
# working directory to the download data directory.
|
56
|
+
#
|
57
|
+
# @yieldparam download_data_dir [String] the path to the download data directory
|
58
|
+
#
|
59
|
+
# @return [void]
|
60
|
+
#
|
61
|
+
# @example
|
62
|
+
# CircleCI::Parallel.configure do |config|
|
63
|
+
# config.on_master_node.after_download do
|
64
|
+
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
65
|
+
# data = JSON.parse(File.read(path))
|
66
|
+
# node_name = File.dirname(path)
|
67
|
+
# merged_data[node_name] = data
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# File.write('merged_data.json', JSON.generate(merged_data))
|
71
|
+
# end
|
72
|
+
# end
|
73
|
+
#
|
74
|
+
# @see CircleCI::Parallel.download_data_dir
|
75
|
+
def after_download(chdir: true, &block)
|
76
|
+
@after_download_hook = Hook.new(block, chdir)
|
77
|
+
end
|
78
|
+
|
79
|
+
# Defines a callback that will be invoked on the master node after syncing all nodes.
|
80
|
+
#
|
81
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
82
|
+
# working directory to the local data directory.
|
83
|
+
#
|
84
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
85
|
+
#
|
86
|
+
# @return [void]
|
87
|
+
#
|
88
|
+
# @example
|
89
|
+
# CircleCI::Parallel.configure do |config|
|
90
|
+
# config.on_master_node.after_sync do
|
91
|
+
# clean_some_intermediate_data
|
92
|
+
# end
|
93
|
+
# end
|
94
|
+
#
|
95
|
+
# @see CircleCI::Parallel.local_data_dir
|
96
|
+
def after_sync(chdir: true, &block)
|
97
|
+
@after_sync_hook = Hook.new(block, chdir)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
class SlaveNodeConfiguration
|
7
|
+
# @api private
|
8
|
+
attr_reader :before_sync_hook, :after_sync_hook
|
9
|
+
|
10
|
+
# @api private
|
11
|
+
def initialize
|
12
|
+
@before_sync_hook = @after_sync_hook = Hook.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# Defines a callback that will be invoked on each slave node before syncing all nodes.
|
16
|
+
#
|
17
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
18
|
+
# working directory to the local data directory.
|
19
|
+
#
|
20
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
21
|
+
#
|
22
|
+
# @return [void]
|
23
|
+
#
|
24
|
+
# @example
|
25
|
+
# CircleCI::Parallel.configure do |config|
|
26
|
+
# config.on_each_slave_node.before_sync do
|
27
|
+
# File.write('data.json', JSON.generate(some_data))
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# @see CircleCI::Parallel.local_data_dir
|
32
|
+
def before_sync(chdir: true, &block)
|
33
|
+
@before_sync_hook = Hook.new(block, chdir)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Defines a callback that will be invoked on each slave node after syncing all nodes.
|
37
|
+
#
|
38
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
39
|
+
# working directory to the local data directory.
|
40
|
+
#
|
41
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
42
|
+
#
|
43
|
+
# @return [void]
|
44
|
+
#
|
45
|
+
# @example
|
46
|
+
# CircleCI::Parallel.configure do |config|
|
47
|
+
# config.on_each_slave_node.after_sync do
|
48
|
+
# clean_some_intermediate_data
|
49
|
+
# end
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# @see CircleCI::Parallel.local_data_dir
|
53
|
+
def after_sync(chdir: true, &block)
|
54
|
+
@after_sync_hook = Hook.new(block, chdir)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -27,7 +27,7 @@ module CircleCI
|
|
27
27
|
@current_node ||= Node.new(current_build, ENV['CIRCLE_NODE_INDEX'].to_i)
|
28
28
|
end
|
29
29
|
|
30
|
-
def
|
30
|
+
def sync
|
31
31
|
validate!
|
32
32
|
task.run
|
33
33
|
end
|
@@ -53,15 +53,17 @@ module CircleCI
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def task
|
56
|
-
@task ||=
|
56
|
+
@task ||= current_node.master? ? master_task : slave_task
|
57
57
|
end
|
58
58
|
|
59
|
-
def
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
59
|
+
def master_task
|
60
|
+
klass = configuration.mock_mode ? Task::MockMaster : Task::Master
|
61
|
+
klass.new(current_node, configuration.master_node_configuration)
|
62
|
+
end
|
63
|
+
|
64
|
+
def slave_task
|
65
|
+
klass = configuration.mock_mode ? Task::MockSlave : Task::Slave
|
66
|
+
klass.new(current_node, configuration.slave_node_configuration)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -5,13 +5,14 @@ module CircleCI
|
|
5
5
|
module Task
|
6
6
|
# @api private
|
7
7
|
class Master < Base
|
8
|
-
def run
|
8
|
+
def run # rubocop:disable Metrics/AbcSize
|
9
9
|
create_node_data_dir
|
10
|
-
configuration.
|
11
|
-
|
10
|
+
configuration.before_sync_hook.call(node.data_dir)
|
11
|
+
mark_as_syncing
|
12
|
+
configuration.before_download_hook.call(BASE_DATA_DIR)
|
12
13
|
download_from_slave_nodes
|
13
14
|
configuration.after_download_hook.call(BASE_DATA_DIR)
|
14
|
-
configuration.
|
15
|
+
configuration.after_sync_hook.call(node.data_dir)
|
15
16
|
done
|
16
17
|
end
|
17
18
|
|
@@ -33,7 +34,7 @@ module CircleCI
|
|
33
34
|
|
34
35
|
Downloader = Struct.new(:node) do
|
35
36
|
def ready_for_download?
|
36
|
-
Kernel.system('ssh', node.ssh_host, 'test', '-f',
|
37
|
+
Kernel.system('ssh', node.ssh_host, 'test', '-f', SYNC_MARKER_FILE)
|
37
38
|
end
|
38
39
|
|
39
40
|
def download
|
@@ -7,10 +7,10 @@ module CircleCI
|
|
7
7
|
class Slave < Base
|
8
8
|
def run
|
9
9
|
create_node_data_dir
|
10
|
-
configuration.
|
11
|
-
|
10
|
+
configuration.before_sync_hook.call(node.data_dir)
|
11
|
+
mark_as_syncing
|
12
12
|
wait_for_master_node_to_download
|
13
|
-
configuration.
|
13
|
+
configuration.after_sync_hook.call(node.data_dir)
|
14
14
|
done
|
15
15
|
end
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: circleci-parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuji Nakayama
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-08-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -24,8 +24,8 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.12'
|
27
|
-
description: Provides Ruby APIs for
|
28
|
-
files between the
|
27
|
+
description: Provides Ruby APIs for syncing CircleCI parallel nodes and transferring
|
28
|
+
files between the nodes
|
29
29
|
email:
|
30
30
|
- nkymyj@gmail.com
|
31
31
|
executables: []
|
@@ -48,6 +48,9 @@ files:
|
|
48
48
|
- lib/circleci/parallel.rb
|
49
49
|
- lib/circleci/parallel/build.rb
|
50
50
|
- lib/circleci/parallel/configuration.rb
|
51
|
+
- lib/circleci/parallel/configuration/configuration_collection_proxy.rb
|
52
|
+
- lib/circleci/parallel/configuration/master_node_configuration.rb
|
53
|
+
- lib/circleci/parallel/configuration/slave_node_configuration.rb
|
51
54
|
- lib/circleci/parallel/environment.rb
|
52
55
|
- lib/circleci/parallel/hook.rb
|
53
56
|
- lib/circleci/parallel/node.rb
|
@@ -77,9 +80,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
77
80
|
version: '0'
|
78
81
|
requirements: []
|
79
82
|
rubyforge_project:
|
80
|
-
rubygems_version: 2.6.
|
83
|
+
rubygems_version: 2.6.11
|
81
84
|
signing_key:
|
82
85
|
specification_version: 4
|
83
|
-
summary: Provides Ruby APIs for
|
84
|
-
between the
|
86
|
+
summary: Provides Ruby APIs for syncing CircleCI parallel nodes and transferring files
|
87
|
+
between the nodes
|
85
88
|
test_files: []
|