circleci-parallel 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -0
- data/CHANGELOG.md +15 -1
- data/README.md +6 -6
- data/Rakefile +2 -2
- data/circleci-parallel.gemspec +3 -2
- data/lib/circleci/parallel.rb +29 -18
- data/lib/circleci/parallel/configuration.rb +36 -66
- data/lib/circleci/parallel/configuration/configuration_collection_proxy.rb +64 -0
- data/lib/circleci/parallel/configuration/master_node_configuration.rb +102 -0
- data/lib/circleci/parallel/configuration/slave_node_configuration.rb +59 -0
- data/lib/circleci/parallel/environment.rb +10 -8
- data/lib/circleci/parallel/task/base.rb +2 -2
- data/lib/circleci/parallel/task/master.rb +6 -5
- data/lib/circleci/parallel/task/slave.rb +3 -3
- data/lib/circleci/parallel/version.rb +2 -2
- metadata +10 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc704707586c01c1552e51c7f025494025a3719f
|
4
|
+
data.tar.gz: 0edc41e83e78e85f9925b1aa0a640fb832372800
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad1a77fa741b4e494c1eef4c4af017810546748aaee18d987fbb00c5225924adf334a6e74829260a78e3c7a05eb8b65e591b2af8bb735086c09106c58861fb82
|
7
|
+
data.tar.gz: f356cb223fc9ba5b9f5ab1e7220b834945d6798304a6e678e8a8b1e0b926d2bb6fc06e0491f6bae2ab5cdb95542401a60ea603ab8fcaaf18a66a801802ee6623
|
data/.rubocop.yml
CHANGED
@@ -1,9 +1,21 @@
|
|
1
1
|
AllCops:
|
2
2
|
TargetRubyVersion: 2.1
|
3
3
|
|
4
|
+
Layout/MultilineMethodCallIndentation:
|
5
|
+
Exclude:
|
6
|
+
- spec/**/*
|
7
|
+
|
8
|
+
Lint/AmbiguousBlockAssociation:
|
9
|
+
Exclude:
|
10
|
+
- spec/**/*
|
11
|
+
|
4
12
|
Metrics/AbcSize:
|
5
13
|
Max: 17
|
6
14
|
|
15
|
+
Metrics/BlockLength:
|
16
|
+
Exclude:
|
17
|
+
- spec/**/*
|
18
|
+
|
7
19
|
Metrics/LineLength:
|
8
20
|
Max: 120
|
9
21
|
|
data/CHANGELOG.md
CHANGED
@@ -2,13 +2,27 @@
|
|
2
2
|
|
3
3
|
## Development
|
4
4
|
|
5
|
+
## v0.5.0
|
6
|
+
|
7
|
+
* Add improved hook configuration APIs.
|
8
|
+
* `config.on_every_node.before_sync`
|
9
|
+
* `config.on_every_node.after_sync`
|
10
|
+
* `config.on_master_node.before_sync`
|
11
|
+
* `config.on_master_node.after_sync`
|
12
|
+
* `config.on_master_node.before_download`
|
13
|
+
* `config.on_master_node.after_download`
|
14
|
+
* `config.on_each_slave_node.before_sync`
|
15
|
+
* `config.on_each_slave_node.after_sync`
|
16
|
+
* Deprecate `#before_join`, `#after_join`, and `#after_download` in `Configuration`.
|
17
|
+
* Add `Parallel.sync` and deprecate `Parallel.join`.
|
18
|
+
|
5
19
|
## v0.4.1
|
6
20
|
|
7
21
|
* Tweak UI messsages.
|
8
22
|
|
9
23
|
## v0.4.0
|
10
24
|
|
11
|
-
* Print "Done." when
|
25
|
+
* Print "Done." when sync is complete.
|
12
26
|
|
13
27
|
## v0.3.0
|
14
28
|
|
data/README.md
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
# CircleCI::Parallel
|
8
8
|
|
9
|
-
**CircleCI::Parallel** provides simple APIs for syncing [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
|
9
|
+
**CircleCI::Parallel** provides simple Ruby APIs for syncing [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
|
10
10
|
and transferring files between the nodes.
|
11
11
|
|
12
12
|
## Installation
|
@@ -47,10 +47,10 @@ test:
|
|
47
47
|
require 'circleci/parallel'
|
48
48
|
|
49
49
|
CircleCI::Parallel.configure do |config|
|
50
|
-
# This hook will be invoked on
|
50
|
+
# This hook will be invoked on every node.
|
51
51
|
# The current working directory in this hook is set to the local data directory
|
52
52
|
# where node specific data should be saved in.
|
53
|
-
config.
|
53
|
+
config.on_every_node.before_sync do
|
54
54
|
data = do_something
|
55
55
|
json = JSON.generate(data)
|
56
56
|
File.write('data.json', json)
|
@@ -68,7 +68,7 @@ CircleCI::Parallel.configure do |config|
|
|
68
68
|
# │ └── node_specific_data_you_saved_on_node1.txt
|
69
69
|
# └── node2
|
70
70
|
# └── node_specific_data_you_saved_on_node2.txt
|
71
|
-
config.after_download do
|
71
|
+
config.on_master_node.after_download do
|
72
72
|
merged_data = {}
|
73
73
|
|
74
74
|
Dir.glob('*/data.json') do |path|
|
@@ -83,8 +83,8 @@ CircleCI::Parallel.configure do |config|
|
|
83
83
|
end
|
84
84
|
|
85
85
|
# Sync all nodes in the same build and gather all node data into the master node.
|
86
|
-
# Invoking this method blocks until the
|
87
|
-
CircleCI::Parallel.
|
86
|
+
# Invoking this method blocks until the sync and the data transfer is complete.
|
87
|
+
CircleCI::Parallel.sync
|
88
88
|
```
|
89
89
|
|
90
90
|
See [API docs](http://www.rubydoc.info/gems/circleci-parallel) for more details.
|
data/Rakefile
CHANGED
data/circleci-parallel.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'circleci/parallel/version'
|
@@ -9,8 +10,8 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ['Yuji Nakayama']
|
10
11
|
spec.email = ['nkymyj@gmail.com']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
'and
|
13
|
+
spec.summary = 'Provides Ruby APIs for syncing CircleCI parallel nodes ' \
|
14
|
+
'and transferring files between the nodes'
|
14
15
|
spec.description = spec.summary
|
15
16
|
spec.homepage = 'https://github.com/increments/circleci-parallel'
|
16
17
|
spec.license = 'MIT'
|
data/lib/circleci/parallel.rb
CHANGED
@@ -3,20 +3,20 @@ require 'forwardable'
|
|
3
3
|
require 'circleci/parallel/environment'
|
4
4
|
|
5
5
|
module CircleCI
|
6
|
-
# Provides simple APIs for
|
7
|
-
#
|
6
|
+
# Provides simple APIs for syncing CircleCI parallel nodes and transferring files between the
|
7
|
+
# nodes.
|
8
8
|
#
|
9
9
|
# @example
|
10
10
|
# merged_data = {}
|
11
11
|
#
|
12
12
|
# CircleCI::Parallel.configure do |config|
|
13
|
-
# config.
|
13
|
+
# config.on_every_node.before_sync do
|
14
14
|
# data = do_something
|
15
15
|
# json = JSON.generate(data)
|
16
16
|
# File.write('data.json', json)
|
17
17
|
# end
|
18
18
|
#
|
19
|
-
# config.after_download do
|
19
|
+
# config.on_master_node.after_download do
|
20
20
|
# Dir.glob('*/data.json') do |path|
|
21
21
|
# json = File.read(path)
|
22
22
|
# data = JSON.parse(json)
|
@@ -26,7 +26,7 @@ module CircleCI
|
|
26
26
|
# end
|
27
27
|
# end
|
28
28
|
#
|
29
|
-
# CircleCI::Parallel.
|
29
|
+
# CircleCI::Parallel.sync
|
30
30
|
#
|
31
31
|
# p merged_data
|
32
32
|
module Parallel
|
@@ -39,7 +39,7 @@ module CircleCI
|
|
39
39
|
BASE_DATA_DIR = File.join(WORK_DIR, 'data')
|
40
40
|
|
41
41
|
# @api private
|
42
|
-
|
42
|
+
SYNC_MARKER_FILE = File.join(WORK_DIR, 'SYNCING')
|
43
43
|
|
44
44
|
# @api private
|
45
45
|
DOWNLOAD_MARKER_FILE = File.join(WORK_DIR, 'DOWNLOADED')
|
@@ -59,7 +59,7 @@ module CircleCI
|
|
59
59
|
#
|
60
60
|
# @!scope class
|
61
61
|
#
|
62
|
-
# Provides a block for configuring
|
62
|
+
# Provides a block for configuring CircleCI::Parallel.
|
63
63
|
#
|
64
64
|
# @yieldparam config [Configuration] the current configuration
|
65
65
|
#
|
@@ -90,24 +90,27 @@ module CircleCI
|
|
90
90
|
#
|
91
91
|
# Returns the current CircleCI node.
|
92
92
|
#
|
93
|
-
# @return [
|
93
|
+
# @return [Node] the current node
|
94
94
|
#
|
95
95
|
# @see .current_build
|
96
96
|
def_delegator :environment, :current_node
|
97
97
|
|
98
|
-
# @!method
|
98
|
+
# @!method sync
|
99
99
|
#
|
100
100
|
# @!scope class
|
101
101
|
#
|
102
|
-
#
|
103
|
-
# Invoking this method blocks until the
|
102
|
+
# Sync all nodes in the same build and gather all node data into the master node.
|
103
|
+
# Invoking this method blocks until the sync and data downloads are complete.
|
104
104
|
#
|
105
105
|
# @raise [RuntimeError] when `CIRCLECI` environment variable is not set
|
106
106
|
#
|
107
|
-
# @see CircleCI::Parallel::
|
108
|
-
# @see CircleCI::Parallel::
|
109
|
-
# @see CircleCI::Parallel::
|
110
|
-
|
107
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_sync
|
108
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_download
|
109
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_download
|
110
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_sync
|
111
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#before_sync
|
112
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#after_sync
|
113
|
+
def_delegator :environment, :sync
|
111
114
|
|
112
115
|
# @api private
|
113
116
|
# @!method puts
|
@@ -123,8 +126,10 @@ module CircleCI
|
|
123
126
|
# path = File.join(CircleCI::Parallel.local_data_dir, 'data.json')
|
124
127
|
# File.write(path, JSON.generate(some_data))
|
125
128
|
#
|
126
|
-
# @see CircleCI::Parallel::
|
127
|
-
# @see CircleCI::Parallel::
|
129
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_sync
|
130
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_sync
|
131
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#before_sync
|
132
|
+
# @see CircleCI::Parallel::SlaveNodeConfiguration#after_sync
|
128
133
|
def local_data_dir
|
129
134
|
current_node.data_dir.tap do |path|
|
130
135
|
FileUtils.makedirs(path) unless Dir.exist?(path)
|
@@ -155,7 +160,8 @@ module CircleCI
|
|
155
160
|
# end
|
156
161
|
# end
|
157
162
|
#
|
158
|
-
# @see CircleCI::Parallel::
|
163
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#before_download
|
164
|
+
# @see CircleCI::Parallel::MasterNodeConfiguration#after_download
|
159
165
|
def download_data_dir
|
160
166
|
BASE_DATA_DIR.tap do |path|
|
161
167
|
FileUtils.makedirs(path) unless Dir.exist?(path)
|
@@ -168,6 +174,11 @@ module CircleCI
|
|
168
174
|
@environment = nil
|
169
175
|
end
|
170
176
|
|
177
|
+
# @deprecated Use `.sync` instead.
|
178
|
+
def join
|
179
|
+
sync
|
180
|
+
end
|
181
|
+
|
171
182
|
private
|
172
183
|
|
173
184
|
def environment
|
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'circleci/parallel/configuration/configuration_collection_proxy'
|
2
|
+
require 'circleci/parallel/configuration/master_node_configuration'
|
3
|
+
require 'circleci/parallel/configuration/slave_node_configuration'
|
1
4
|
require 'circleci/parallel/hook'
|
2
5
|
|
3
6
|
module CircleCI
|
@@ -9,83 +12,50 @@ module CircleCI
|
|
9
12
|
# @return [Boolean] whether mock mode is enabled (default: false)
|
10
13
|
attr_accessor :mock_mode
|
11
14
|
|
12
|
-
# @api private
|
13
|
-
attr_reader :before_join_hook, :after_join_hook, :after_download_hook
|
14
|
-
|
15
15
|
def initialize
|
16
16
|
@silent = false
|
17
17
|
@mock_mode = false
|
18
|
-
@before_join_hook = @after_join_hook = @after_download_hook = Hook.new
|
19
18
|
end
|
20
19
|
|
21
|
-
#
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
#
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
# @
|
20
|
+
# @return [ConfigurationCollectionProxy]
|
21
|
+
def on_every_node
|
22
|
+
@every_node_configuration ||=
|
23
|
+
ConfigurationCollectionProxy.new(master_node_configuration, slave_node_configuration)
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [MasterNodeConfiguration]
|
27
|
+
def on_master_node
|
28
|
+
master_node_configuration
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [SlaveNodeConfiguration]
|
32
|
+
def on_each_slave_node
|
33
|
+
slave_node_configuration
|
34
|
+
end
|
35
|
+
|
36
|
+
# @deprecated Use `on_every_node.before_sync` instead.
|
38
37
|
def before_join(chdir: true, &block)
|
39
|
-
|
38
|
+
on_every_node.before_sync(chdir: chdir, &block)
|
40
39
|
end
|
41
40
|
|
42
|
-
#
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
# @
|
48
|
-
#
|
49
|
-
# @return [void]
|
50
|
-
#
|
51
|
-
# @example
|
52
|
-
# CircleCI::Parallel.configure do |config|
|
53
|
-
# config.after_join do
|
54
|
-
# clean_some_intermediate_data
|
55
|
-
# end
|
56
|
-
# end
|
57
|
-
#
|
58
|
-
# @see CircleCI::Parallel.local_data_dir
|
41
|
+
# @deprecated Use `on_master_node.after_download` instead.
|
42
|
+
def after_download(chdir: true, &block)
|
43
|
+
on_master_node.after_download(chdir: chdir, &block)
|
44
|
+
end
|
45
|
+
|
46
|
+
# @deprecated Use `on_every_node.after_sync` instead.
|
59
47
|
def after_join(chdir: true, &block)
|
60
|
-
|
48
|
+
on_every_node.after_sync(chdir: chdir, &block)
|
61
49
|
end
|
62
50
|
|
63
|
-
#
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
#
|
69
|
-
|
70
|
-
|
71
|
-
# @return [void]
|
72
|
-
#
|
73
|
-
# @example
|
74
|
-
# CircleCI::Parallel.configure do |config|
|
75
|
-
# config.after_download do
|
76
|
-
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
77
|
-
# data = JSON.parse(File.read(path))
|
78
|
-
# node_name = File.dirname(path)
|
79
|
-
# merged_data[node_name] = data
|
80
|
-
# end
|
81
|
-
#
|
82
|
-
# File.write('merged_data.json', JSON.generate(merged_data))
|
83
|
-
# end
|
84
|
-
# end
|
85
|
-
#
|
86
|
-
# @see CircleCI::Parallel.download_data_dir
|
87
|
-
def after_download(chdir: true, &block)
|
88
|
-
@after_download_hook = Hook.new(block, chdir)
|
51
|
+
# @api private
|
52
|
+
def master_node_configuration
|
53
|
+
@master_node_configuration ||= MasterNodeConfiguration.new
|
54
|
+
end
|
55
|
+
|
56
|
+
# @api private
|
57
|
+
def slave_node_configuration
|
58
|
+
@slave_node_configuration ||= SlaveNodeConfiguration.new
|
89
59
|
end
|
90
60
|
end
|
91
61
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
# A convenient proxy to modify both master node and slave node configuration at once.
|
7
|
+
class ConfigurationCollectionProxy
|
8
|
+
# @api private
|
9
|
+
attr_reader :configurations
|
10
|
+
|
11
|
+
# @api private
|
12
|
+
def initialize(*configurations)
|
13
|
+
@configurations = configurations
|
14
|
+
end
|
15
|
+
|
16
|
+
# Defines a callback that will be invoked on every node before syncing all nodes.
|
17
|
+
#
|
18
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
19
|
+
# working directory to the local data directory.
|
20
|
+
#
|
21
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
22
|
+
#
|
23
|
+
# @return [void]
|
24
|
+
#
|
25
|
+
# @example
|
26
|
+
# CircleCI::Parallel.configure do |config|
|
27
|
+
# config.on_every_node.before_sync do
|
28
|
+
# File.write('data.json', JSON.generate(some_data))
|
29
|
+
# end
|
30
|
+
# end
|
31
|
+
#
|
32
|
+
# @see CircleCI::Parallel.local_data_dir
|
33
|
+
def before_sync(chdir: true, &block)
|
34
|
+
configurations.each do |configuration|
|
35
|
+
configuration.before_sync(chdir: chdir, &block)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Defines a callback that will be invoked on every node after syncing all nodes.
|
40
|
+
#
|
41
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
42
|
+
# working directory to the local data directory.
|
43
|
+
#
|
44
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
45
|
+
#
|
46
|
+
# @return [void]
|
47
|
+
#
|
48
|
+
# @example
|
49
|
+
# CircleCI::Parallel.configure do |config|
|
50
|
+
# config.on_each_slave_node.after_sync do
|
51
|
+
# clean_some_intermediate_data
|
52
|
+
# end
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# @see CircleCI::Parallel.local_data_dir
|
56
|
+
def after_sync(chdir: true, &block)
|
57
|
+
configurations.each do |configuration|
|
58
|
+
configuration.after_sync(chdir: chdir, &block)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
class MasterNodeConfiguration
|
7
|
+
# @api private
|
8
|
+
attr_reader :before_sync_hook, :before_download_hook, :after_download_hook, :after_sync_hook
|
9
|
+
|
10
|
+
# @api private
|
11
|
+
def initialize
|
12
|
+
@before_sync_hook = @before_download_hook = @after_download_hook = @after_sync_hook = Hook.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# Defines a callback that will be invoked on the master node before syncing all nodes.
|
16
|
+
#
|
17
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
18
|
+
# working directory to the local data directory.
|
19
|
+
#
|
20
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
21
|
+
#
|
22
|
+
# @return [void]
|
23
|
+
#
|
24
|
+
# @example
|
25
|
+
# CircleCI::Parallel.configure do |config|
|
26
|
+
# config.on_master_node.before_sync do
|
27
|
+
# File.write('data.json', JSON.generate(some_data))
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# @see CircleCI::Parallel.local_data_dir
|
32
|
+
def before_sync(chdir: true, &block)
|
33
|
+
@before_sync_hook = Hook.new(block, chdir)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Defines a callback that will be invoked on the master node before downloading all data
|
37
|
+
# from slave nodes.
|
38
|
+
#
|
39
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
40
|
+
# working directory to the download data directory.
|
41
|
+
#
|
42
|
+
# @yieldparam download_data_dir [String] the path to the download data directory
|
43
|
+
#
|
44
|
+
# @return [void]
|
45
|
+
#
|
46
|
+
# @see CircleCI::Parallel.download_data_dir
|
47
|
+
def before_download(chdir: true, &block)
|
48
|
+
@before_download_hook = Hook.new(block, chdir)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Defines a callback that will be invoked on the master node after downloading all data
|
52
|
+
# from slave nodes.
|
53
|
+
#
|
54
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
55
|
+
# working directory to the download data directory.
|
56
|
+
#
|
57
|
+
# @yieldparam download_data_dir [String] the path to the download data directory
|
58
|
+
#
|
59
|
+
# @return [void]
|
60
|
+
#
|
61
|
+
# @example
|
62
|
+
# CircleCI::Parallel.configure do |config|
|
63
|
+
# config.on_master_node.after_download do
|
64
|
+
# merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
|
65
|
+
# data = JSON.parse(File.read(path))
|
66
|
+
# node_name = File.dirname(path)
|
67
|
+
# merged_data[node_name] = data
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# File.write('merged_data.json', JSON.generate(merged_data))
|
71
|
+
# end
|
72
|
+
# end
|
73
|
+
#
|
74
|
+
# @see CircleCI::Parallel.download_data_dir
|
75
|
+
def after_download(chdir: true, &block)
|
76
|
+
@after_download_hook = Hook.new(block, chdir)
|
77
|
+
end
|
78
|
+
|
79
|
+
# Defines a callback that will be invoked on the master node after syncing all nodes.
|
80
|
+
#
|
81
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
82
|
+
# working directory to the local data directory.
|
83
|
+
#
|
84
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
85
|
+
#
|
86
|
+
# @return [void]
|
87
|
+
#
|
88
|
+
# @example
|
89
|
+
# CircleCI::Parallel.configure do |config|
|
90
|
+
# config.on_master_node.after_sync do
|
91
|
+
# clean_some_intermediate_data
|
92
|
+
# end
|
93
|
+
# end
|
94
|
+
#
|
95
|
+
# @see CircleCI::Parallel.local_data_dir
|
96
|
+
def after_sync(chdir: true, &block)
|
97
|
+
@after_sync_hook = Hook.new(block, chdir)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'circleci/parallel/hook'
|
2
|
+
|
3
|
+
module CircleCI
|
4
|
+
module Parallel
|
5
|
+
class Configuration
|
6
|
+
class SlaveNodeConfiguration
|
7
|
+
# @api private
|
8
|
+
attr_reader :before_sync_hook, :after_sync_hook
|
9
|
+
|
10
|
+
# @api private
|
11
|
+
def initialize
|
12
|
+
@before_sync_hook = @after_sync_hook = Hook.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# Defines a callback that will be invoked on each slave node before syncing all nodes.
|
16
|
+
#
|
17
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
18
|
+
# working directory to the local data directory.
|
19
|
+
#
|
20
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
21
|
+
#
|
22
|
+
# @return [void]
|
23
|
+
#
|
24
|
+
# @example
|
25
|
+
# CircleCI::Parallel.configure do |config|
|
26
|
+
# config.on_each_slave_node.before_sync do
|
27
|
+
# File.write('data.json', JSON.generate(some_data))
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# @see CircleCI::Parallel.local_data_dir
|
32
|
+
def before_sync(chdir: true, &block)
|
33
|
+
@before_sync_hook = Hook.new(block, chdir)
|
34
|
+
end
|
35
|
+
|
36
|
+
# Defines a callback that will be invoked on each slave node after syncing all nodes.
|
37
|
+
#
|
38
|
+
# @param chdir [Boolean] whether the callback should be invoked while changing the current
|
39
|
+
# working directory to the local data directory.
|
40
|
+
#
|
41
|
+
# @yieldparam local_data_dir [String] the path to the local data directory
|
42
|
+
#
|
43
|
+
# @return [void]
|
44
|
+
#
|
45
|
+
# @example
|
46
|
+
# CircleCI::Parallel.configure do |config|
|
47
|
+
# config.on_each_slave_node.after_sync do
|
48
|
+
# clean_some_intermediate_data
|
49
|
+
# end
|
50
|
+
# end
|
51
|
+
#
|
52
|
+
# @see CircleCI::Parallel.local_data_dir
|
53
|
+
def after_sync(chdir: true, &block)
|
54
|
+
@after_sync_hook = Hook.new(block, chdir)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -27,7 +27,7 @@ module CircleCI
|
|
27
27
|
@current_node ||= Node.new(current_build, ENV['CIRCLE_NODE_INDEX'].to_i)
|
28
28
|
end
|
29
29
|
|
30
|
-
def
|
30
|
+
def sync
|
31
31
|
validate!
|
32
32
|
task.run
|
33
33
|
end
|
@@ -53,15 +53,17 @@ module CircleCI
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def task
|
56
|
-
@task ||=
|
56
|
+
@task ||= current_node.master? ? master_task : slave_task
|
57
57
|
end
|
58
58
|
|
59
|
-
def
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
59
|
+
def master_task
|
60
|
+
klass = configuration.mock_mode ? Task::MockMaster : Task::Master
|
61
|
+
klass.new(current_node, configuration.master_node_configuration)
|
62
|
+
end
|
63
|
+
|
64
|
+
def slave_task
|
65
|
+
klass = configuration.mock_mode ? Task::MockSlave : Task::Slave
|
66
|
+
klass.new(current_node, configuration.slave_node_configuration)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -5,13 +5,14 @@ module CircleCI
|
|
5
5
|
module Task
|
6
6
|
# @api private
|
7
7
|
class Master < Base
|
8
|
-
def run
|
8
|
+
def run # rubocop:disable Metrics/AbcSize
|
9
9
|
create_node_data_dir
|
10
|
-
configuration.
|
11
|
-
|
10
|
+
configuration.before_sync_hook.call(node.data_dir)
|
11
|
+
mark_as_syncing
|
12
|
+
configuration.before_download_hook.call(BASE_DATA_DIR)
|
12
13
|
download_from_slave_nodes
|
13
14
|
configuration.after_download_hook.call(BASE_DATA_DIR)
|
14
|
-
configuration.
|
15
|
+
configuration.after_sync_hook.call(node.data_dir)
|
15
16
|
done
|
16
17
|
end
|
17
18
|
|
@@ -33,7 +34,7 @@ module CircleCI
|
|
33
34
|
|
34
35
|
Downloader = Struct.new(:node) do
|
35
36
|
def ready_for_download?
|
36
|
-
Kernel.system('ssh', node.ssh_host, 'test', '-f',
|
37
|
+
Kernel.system('ssh', node.ssh_host, 'test', '-f', SYNC_MARKER_FILE)
|
37
38
|
end
|
38
39
|
|
39
40
|
def download
|
@@ -7,10 +7,10 @@ module CircleCI
|
|
7
7
|
class Slave < Base
|
8
8
|
def run
|
9
9
|
create_node_data_dir
|
10
|
-
configuration.
|
11
|
-
|
10
|
+
configuration.before_sync_hook.call(node.data_dir)
|
11
|
+
mark_as_syncing
|
12
12
|
wait_for_master_node_to_download
|
13
|
-
configuration.
|
13
|
+
configuration.after_sync_hook.call(node.data_dir)
|
14
14
|
done
|
15
15
|
end
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: circleci-parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuji Nakayama
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-08-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -24,8 +24,8 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.12'
|
27
|
-
description: Provides Ruby APIs for
|
28
|
-
files between the
|
27
|
+
description: Provides Ruby APIs for syncing CircleCI parallel nodes and transferring
|
28
|
+
files between the nodes
|
29
29
|
email:
|
30
30
|
- nkymyj@gmail.com
|
31
31
|
executables: []
|
@@ -48,6 +48,9 @@ files:
|
|
48
48
|
- lib/circleci/parallel.rb
|
49
49
|
- lib/circleci/parallel/build.rb
|
50
50
|
- lib/circleci/parallel/configuration.rb
|
51
|
+
- lib/circleci/parallel/configuration/configuration_collection_proxy.rb
|
52
|
+
- lib/circleci/parallel/configuration/master_node_configuration.rb
|
53
|
+
- lib/circleci/parallel/configuration/slave_node_configuration.rb
|
51
54
|
- lib/circleci/parallel/environment.rb
|
52
55
|
- lib/circleci/parallel/hook.rb
|
53
56
|
- lib/circleci/parallel/node.rb
|
@@ -77,9 +80,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
77
80
|
version: '0'
|
78
81
|
requirements: []
|
79
82
|
rubyforge_project:
|
80
|
-
rubygems_version: 2.6.
|
83
|
+
rubygems_version: 2.6.11
|
81
84
|
signing_key:
|
82
85
|
specification_version: 4
|
83
|
-
summary: Provides Ruby APIs for
|
84
|
-
between the
|
86
|
+
summary: Provides Ruby APIs for syncing CircleCI parallel nodes and transferring files
|
87
|
+
between the nodes
|
85
88
|
test_files: []
|