circleci-parallel 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a8f45e02994f05608bc9d43299f08a24fad83822
4
+ data.tar.gz: 1eacbf5a19c69c374ef56114053428f98ed83601
5
+ SHA512:
6
+ metadata.gz: b7b266accba3b7152ed128e07306bd04edf985819600951bf2c38437cd23d7d7e6e273d39850f86f0c3814247af35acfa83e8a3ac2876f8b975b74feb757166e
7
+ data.tar.gz: 2e8d863583c3d3195690f413945a0495d4ab1cbc57b758ca5a9f79b6a741b6267458c5b82e3cb4a70a14f7e1f9a8a1994079bdc9c273197fe2b3ba0ac9132d1f
@@ -0,0 +1,10 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/examples.txt
9
+ /spec/reports/
10
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --require spec_helper
@@ -0,0 +1,35 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.1
3
+
4
+ Metrics/AbcSize:
5
+ Max: 16
6
+
7
+ Metrics/LineLength:
8
+ Max: 120
9
+
10
+ Style/AsciiComments:
11
+ Exclude:
12
+ - lib/circleci/parallel.rb
13
+
14
+ Style/ClassAndModuleChildren:
15
+ Exclude:
16
+ - spec/**/*
17
+
18
+ Style/Documentation:
19
+ Enabled: false
20
+
21
+ Style/EmptyElse:
22
+ Enabled: false
23
+
24
+ Style/GuardClause:
25
+ Enabled: false
26
+
27
+ Style/IndentArray:
28
+ EnforcedStyle: consistent
29
+
30
+ Style/RegexpLiteral:
31
+ Exclude:
32
+ - '*.gemspec'
33
+
34
+ Style/SingleLineBlockParams:
35
+ Enabled: false
@@ -0,0 +1,2 @@
1
+ --markup markdown
2
+ --hide-void-return
data/Gemfile ADDED
@@ -0,0 +1,11 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ group :development, :test do
6
+ gem 'pry-byebug', '~> 3.4'
7
+ gem 'rake', '~> 11.0'
8
+ gem 'rspec', '~> 3.5'
9
+ gem 'rubocop', '~> 0.42'
10
+ gem 'yard', '~> 0.9'
11
+ end
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 Yuji Nakayama
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,86 @@
1
+ # CircleCI::Parallel
2
+
3
+ **CircleCI::Parallel** provides simple APIs for joining [CircleCI parallel nodes](https://circleci.com/docs/parallelism/)
4
+ and sharing files between the nodes.
5
+
6
+ ## Installation
7
+
8
+ Add this line to your application's Gemfile:
9
+
10
+ ```ruby
11
+ gem 'circleci-parallel'
12
+ ```
13
+
14
+ And then execute:
15
+
16
+ ```
17
+ $ bundle install
18
+ ```
19
+
20
+ ## Basic Usage
21
+
22
+ Before using CircleCI::Parallel:
23
+
24
+ * [Add `parallel: true`](https://circleci.com/docs/parallel-manual-setup/)
25
+ to the command that you'll use CircleCI::Parallel in your `circle.yml`.
26
+ * [Set up parallelism](https://circleci.com/docs/setting-up-parallelism/)
27
+ for your project from the CircleCI web console.
28
+
29
+ CircleCI::Parallel uses SSH for joining and transferring data between nodes.
30
+
31
+ ```yaml
32
+ # circle.yml
33
+ test:
34
+ override:
35
+ - ruby test.rb:
36
+ parallel: true
37
+ ```
38
+
39
+ ```ruby
40
+ # test.rb
41
+ require 'circleci/parallel'
42
+
43
+ merged_data = {}
44
+
45
+ CircleCI::Parallel.configure do |config|
46
+ # This hook will be invoked on all the nodes.
47
+ # The current working directory in this hook is set to the local data directory
48
+ # where node specific data should be saved in.
49
+ config.before_join do
50
+ data = do_something
51
+ json = JSON.generate(data)
52
+ File.write('data.json', json)
53
+ end
54
+
55
+ # This hook will be invoked only on the master node after downloading all data from slave nodes.
56
+ # The current working directory in this hook is set to the download data directory
57
+ # where all node data are gathered into.
58
+ # The directory structure on the master node will be the following:
59
+ #
60
+ # .
61
+ # ├── node0
62
+ # │   └── node_specific_data_you_saved_on_node0.txt
63
+ # ├── node1
64
+ # │   └── node_specific_data_you_saved_on_node1.txt
65
+ # └── node2
66
+ #    └── node_specific_data_you_saved_on_node2.txt
67
+ config.after_download do
68
+ Dir.glob('*/data.json') do |path|
69
+ json = File.read(path)
70
+ data = JSON.parse(json)
71
+ node_name = File.dirname(path)
72
+ merged_data[node_name] = data
73
+ end
74
+ end
75
+ end
76
+
77
+ # Join all nodes in the same build and gather all node data into the master node.
78
+ # Invoking this method blocks until the join and data downloads are complete.
79
+ CircleCI::Parallel.join
80
+
81
+ p merged_data
82
+ ```
83
+
84
+ ## License
85
+
86
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,16 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+ require 'rubocop/rake_task'
4
+ require 'yard'
5
+
6
+ RSpec::Core::RakeTask.new do |task|
7
+ task.verbose = false
8
+ end
9
+
10
+ RuboCop::RakeTask.new
11
+
12
+ YARD::Rake::YardocTask.new
13
+
14
+ task default: [:spec, :rubocop]
15
+
16
+ task ci: [:spec, :rubocop]
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require 'circleci/parallel'
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ require 'pry'
10
+ Pry.start
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,7 @@
1
+ machine:
2
+ ruby:
3
+ version: 2.3.1
4
+ test:
5
+ override:
6
+ - bundle exec rake ci:
7
+ parallel: true
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'circleci/parallel/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'circleci-parallel'
8
+ spec.version = CircleCI::Parallel::Version.to_s
9
+ spec.authors = ['Yuji Nakayama']
10
+ spec.email = ['nkymyj@gmail.com']
11
+
12
+ spec.summary = "Provides Ruby APIs for joining CircleCI's parallel builds " \
13
+ 'and sharing files between the builds'
14
+ spec.description = spec.summary
15
+ spec.homepage = 'https://github.com/increments/circleci-parallel'
16
+ spec.license = 'MIT'
17
+
18
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^spec/}) }
19
+ spec.bindir = 'exe'
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ['lib']
22
+
23
+ spec.add_development_dependency 'bundler', '~> 1.12'
24
+ end
@@ -0,0 +1,178 @@
1
+ require 'fileutils'
2
+ require 'forwardable'
3
+ require 'circleci/parallel/environment'
4
+
5
+ module CircleCI
6
+ # Provides simple APIs for joining CircleCI's parallel builds and sharing files between the
7
+ # builds.
8
+ #
9
+ # @example
10
+ # merged_data = {}
11
+ #
12
+ # CircleCI::Parallel.configure do |config|
13
+ # config.before_join do
14
+ # data = do_something
15
+ # json = JSON.generate(data)
16
+ # File.write('data.json', json)
17
+ # end
18
+ #
19
+ # config.after_download do
20
+ # Dir.glob('*/data.json') do |path|
21
+ # json = File.read(path)
22
+ # data = JSON.parse(json)
23
+ # node_name = File.dirname(path)
24
+ # merged_data[node_name] = data
25
+ # end
26
+ # end
27
+ # end
28
+ #
29
+ # CircleCI::Parallel.join
30
+ #
31
+ # p merged_data
32
+ module Parallel
33
+ extend SingleForwardable
34
+
35
+ # @api private
36
+ WORK_DIR = '/tmp/circleci-parallel'.freeze
37
+
38
+ # @api private
39
+ BASE_DATA_DIR = File.join(WORK_DIR, 'data')
40
+
41
+ # @api private
42
+ JOIN_MARKER_FILE = File.join(WORK_DIR, 'JOINING')
43
+
44
+ # @api private
45
+ DOWNLOAD_MARKER_FILE = File.join(WORK_DIR, 'DOWNLOADED')
46
+
47
+ # @!method configuration
48
+
49
+ # @!scope class
50
+ #
51
+ # Returns the current configuration.
52
+ #
53
+ # @return [Configuration] the current configuration
54
+ #
55
+ # @see .configure
56
+ def_delegator :environment, :configuration
57
+
58
+ # @!method configure
59
+ #
60
+ # @!scope class
61
+ #
62
+ # Provides a block for configuring RSpec::ComposableJSONMatchers.
63
+ #
64
+ # @yieldparam config [Configuration] the current configuration
65
+ #
66
+ # @return [void]
67
+ #
68
+ # @example
69
+ # CircleCI::Parallel.configure do |config|
70
+ # config.silent = true
71
+ # end
72
+ #
73
+ # @see .configuration
74
+ def_delegator :environment, :configure
75
+
76
+ # @!method current_build
77
+ #
78
+ # @!scope class
79
+ #
80
+ # Returns the current CircleCI build.
81
+ #
82
+ # @return [Build] the current build
83
+ #
84
+ # @see .current_node
85
+ def_delegator :environment, :current_build
86
+
87
+ # @!method current_node
88
+ #
89
+ # @!scope class
90
+ #
91
+ # Returns the current CircleCI node.
92
+ #
93
+ # @return [Build] the current node
94
+ #
95
+ # @see .current_build
96
+ def_delegator :environment, :current_node
97
+
98
+ # @!method join
99
+ #
100
+ # @!scope class
101
+ #
102
+ # Join all nodes in the same build and gather all node data into the master node.
103
+ # Invoking this method blocks until the join and data downloads are complete.
104
+ #
105
+ # @raise [RuntimeError] when `CIRCLECI` environment variable is not set
106
+ #
107
+ # @see CircleCI::Parallel::Configuration#before_join
108
+ # @see CircleCI::Parallel::Configuration#after_join
109
+ # @see CircleCI::Parallel::Configuration#after_download
110
+ def_delegator :environment, :join
111
+
112
+ # @api private
113
+ # @!method puts
114
+ # @!scope class
115
+ def_delegator :environment, :puts
116
+
117
+ class << self
118
+ # Returns the local data directory where node specific data should be saved in.
119
+ #
120
+ # @return [String] the local data directory
121
+ #
122
+ # @example
123
+ # path = File.join(CircleCI::Parallel.local_data_dir, 'data.json')
124
+ # File.write(path, JSON.generate(some_data))
125
+ #
126
+ # @see CircleCI::Parallel::Configuration#before_join
127
+ # @see CircleCI::Parallel::Configuration#after_join
128
+ def local_data_dir
129
+ current_node.data_dir.tap do |path|
130
+ FileUtils.makedirs(path) unless Dir.exist?(path)
131
+ end
132
+ end
133
+
134
+ # Returns the download data directory where all node data will be downloaded.
135
+ # Note that only master node downloads data from other slave node.
136
+ # When the downloads are complete, the directory structure on the master node will be the
137
+ # following:
138
+ #
139
+ # .
140
+ # ├── node0
141
+ # │   └── node_specific_data_you_saved_on_node0.txt
142
+ # ├── node1
143
+ # │   └── node_specific_data_you_saved_on_node1.txt
144
+ # └── node2
145
+ #    └── node_specific_data_you_saved_on_node2.txt
146
+ #
147
+ # @return [String] the download data directory
148
+ #
149
+ # @example
150
+ # Dir.chdir(CircleCI::Parallel.download_data_dir) do
151
+ # merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
152
+ # data = JSON.parse(File.read(path))
153
+ # node_name = File.dirname(path)
154
+ # merged_data[node_name] = data
155
+ # end
156
+ # end
157
+ #
158
+ # @see CircleCI::Parallel::Configuration#after_download
159
+ def download_data_dir
160
+ BASE_DATA_DIR.tap do |path|
161
+ FileUtils.makedirs(path) unless Dir.exist?(path)
162
+ end
163
+ end
164
+
165
+ # @api private
166
+ def reset!
167
+ environment.clean
168
+ @environment = nil
169
+ end
170
+
171
+ private
172
+
173
+ def environment
174
+ @environment ||= Environment.new
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,32 @@
1
+ require 'circleci/parallel/node'
2
+
3
+ module CircleCI
4
+ module Parallel
5
+ # Represents a CircleCI build.
6
+ class Build
7
+ attr_reader :number, :node_count
8
+
9
+ # @param number [Integer] the build number (`CIRCLE_BUILD_NUM`)
10
+ # @param node_count [Integer] node count of the build (`CIRCLE_NODE_TOTAL`)
11
+ def initialize(number, node_count)
12
+ @number = number
13
+ @node_count = node_count
14
+ end
15
+
16
+ def ==(other)
17
+ number == other.number
18
+ end
19
+
20
+ alias eql? ==
21
+
22
+ def hash
23
+ number.hash ^ node_count.hash
24
+ end
25
+
26
+ # @return [Array<Node>] nodes of the build
27
+ def nodes
28
+ @nodes ||= Array.new(node_count) { |index| Node.new(self, index) }.freeze
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,88 @@
1
+ require 'circleci/parallel/hook'
2
+
3
+ module CircleCI
4
+ module Parallel
5
+ class Configuration
6
+ # @return [Boolean] whether progress messages should be outputted to STDOUT (default: false)
7
+ attr_accessor :silent
8
+
9
+ # @api private
10
+ attr_reader :before_join_hook, :after_join_hook, :after_download_hook
11
+
12
+ def initialize
13
+ @silent = false
14
+ @before_join_hook = @after_join_hook = @after_download_hook = Hook.new
15
+ end
16
+
17
+ # Defines a callback that will be invoked on all nodes before joining nodes.
18
+ #
19
+ # @param chdir [Boolean] whether the callback should be invoked while chaging the current
20
+ # working directory to the local data directory.
21
+ #
22
+ # @yieldparam local_data_dir [String] the path to the local data directory
23
+ #
24
+ # @return [void]
25
+ #
26
+ # @example
27
+ # CircleCI::Parallel.configure do |config|
28
+ # config.before_join do
29
+ # File.write('data.json', JSON.generate(some_data))
30
+ # end
31
+ # end
32
+ #
33
+ # @see CircleCI::Parallel.local_data_dir
34
+ def before_join(chdir: true, &block)
35
+ @before_join_hook = Hook.new(block, chdir)
36
+ end
37
+
38
+ # Defines a callback that will be invoked on all nodes after joining nodes.
39
+ #
40
+ # @param chdir [Boolean] whether the callback should be invoked while chaging the current
41
+ # working directory to the local data directory.
42
+ #
43
+ # @yieldparam local_data_dir [String] the path to the local data directory
44
+ #
45
+ # @return [void]
46
+ #
47
+ # @example
48
+ # CircleCI::Parallel.configure do |config|
49
+ # config.after_join do
50
+ # clean_some_intermediate_data
51
+ # end
52
+ # end
53
+ #
54
+ # @see CircleCI::Parallel.local_data_dir
55
+ def after_join(chdir: true, &block)
56
+ @after_join_hook = Hook.new(block, chdir)
57
+ end
58
+
59
+ # Defines a callback that will be invoked only on the master node after downloading all data
60
+ # from slave nodes.
61
+ #
62
+ # @param chdir [Boolean] whether the callback should be invoked while chaging the current
63
+ # working directory to the download data directory.
64
+ #
65
+ # @yieldparam download_data_dir [String] the path to the download data directory
66
+ #
67
+ # @return [void]
68
+ #
69
+ # @example
70
+ # CircleCI::Parallel.configure do |config|
71
+ # config.after_download do
72
+ # merged_data = Dir['*/data.json'].each_with_object({}) do |path, merged_data|
73
+ # data = JSON.parse(File.read(path))
74
+ # node_name = File.dirname(path)
75
+ # merged_data[node_name] = data
76
+ # end
77
+ #
78
+ # File.write('merged_data.json', JSON.generate(merged_data))
79
+ # end
80
+ # end
81
+ #
82
+ # @see CircleCI::Parallel.download_data_dir
83
+ def after_download(chdir: true, &block)
84
+ @after_download_hook = Hook.new(block, chdir)
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,61 @@
1
+ require 'fileutils'
2
+ require 'circleci/parallel/build'
3
+ require 'circleci/parallel/configuration'
4
+ require 'circleci/parallel/node'
5
+ require 'circleci/parallel/task/master'
6
+ require 'circleci/parallel/task/slave'
7
+
8
+ module CircleCI
9
+ module Parallel
10
+ # @api private
11
+ class Environment
12
+ def configuration
13
+ @configuration ||= Configuration.new
14
+ end
15
+
16
+ def configure
17
+ yield configuration
18
+ end
19
+
20
+ def current_build
21
+ @current_build ||= Build.new(ENV['CIRCLE_BUILD_NUM'].to_i, ENV['CIRCLE_NODE_TOTAL'].to_i)
22
+ end
23
+
24
+ def current_node
25
+ @current_node ||= Node.new(current_build, ENV['CIRCLE_NODE_INDEX'].to_i)
26
+ end
27
+
28
+ def join
29
+ validate!
30
+ task.run
31
+ end
32
+
33
+ def puts(*args)
34
+ Kernel.puts(*args) unless configuration.silent
35
+ end
36
+
37
+ def clean
38
+ FileUtils.rmtree(WORK_DIR) if Dir.exist?(WORK_DIR)
39
+ end
40
+
41
+ private
42
+
43
+ def validate!
44
+ raise 'The current environment is not on CircleCI.' unless ENV['CIRCLECI']
45
+
46
+ unless ENV['CIRCLE_NODE_TOTAL']
47
+ warn 'Environment variable CIRCLE_NODE_TOTAL is not set. ' \
48
+ 'Maybe you forgot adding `parallel: true` to your circle.yml? ' \
49
+ 'https://circleci.com/docs/parallel-manual-setup/'
50
+ end
51
+ end
52
+
53
+ def task
54
+ @task ||= begin
55
+ task_class = current_node.master? ? Task::Master : Task::Slave
56
+ task_class.new(current_node, configuration)
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,23 @@
1
+ module CircleCI
2
+ module Parallel
3
+ # @api private
4
+ class Hook
5
+ attr_reader :proc, :chdir
6
+
7
+ def initialize(proc = nil, chdir = true)
8
+ @proc = proc
9
+ @chdir = chdir
10
+ end
11
+
12
+ def call(dir)
13
+ return unless proc
14
+
15
+ if chdir
16
+ Dir.chdir(dir, &proc)
17
+ else
18
+ proc.call(dir)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,48 @@
1
+ module CircleCI
2
+ module Parallel
3
+ # Represents a CircleCI node.
4
+ class Node
5
+ attr_reader :build, :index
6
+
7
+ # @param build [Build] the build that the node belongs to
8
+ # @param index [Integer] node index (`CIRCLE_NODE_INDEX`)
9
+ def initialize(build, index)
10
+ @build = build
11
+ @index = index
12
+ end
13
+
14
+ def ==(other)
15
+ build == other.build && index == other.index
16
+ end
17
+
18
+ alias eql? ==
19
+
20
+ def hash
21
+ build.hash ^ index.hash
22
+ end
23
+
24
+ # @return [Boolean] whether the node is the master node or not
25
+ def master?
26
+ index.zero?
27
+ end
28
+
29
+ # @return [String] the hostname that can be used for `ssh` command to connect between nodes
30
+ def ssh_host
31
+ # https://circleci.com/docs/ssh-between-build-containers/
32
+ "node#{index}"
33
+ end
34
+
35
+ # @return [String] the local data directory where node specific data should be saved in
36
+ #
37
+ # @see CircleCI::Parallel.local_data_dir
38
+ def data_dir
39
+ File.join(BASE_DATA_DIR, ssh_host)
40
+ end
41
+
42
+ # @return [Array<Node>] other nodes of the same build
43
+ def other_nodes
44
+ @other_nodes ||= (build.nodes - [self]).freeze
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,32 @@
1
+ require 'fileutils'
2
+
3
+ module CircleCI
4
+ module Parallel
5
+ module Task
6
+ # @api private
7
+ class Base
8
+ attr_reader :node, :configuration
9
+
10
+ def initialize(node, configuration)
11
+ @node = node
12
+ @configuration = configuration
13
+ end
14
+
15
+ def run
16
+ raise NotImplementedError
17
+ end
18
+
19
+ private
20
+
21
+ def create_node_data_dir
22
+ FileUtils.makedirs(node.data_dir)
23
+ end
24
+
25
+ def mark_as_joining
26
+ Parallel.puts('Joining CircleCI nodes...')
27
+ File.write(JOIN_MARKER_FILE, '')
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,61 @@
1
+ require 'circleci/parallel/task/base'
2
+
3
+ module CircleCI
4
+ module Parallel
5
+ module Task
6
+ # @api private
7
+ class Master < Base
8
+ def run
9
+ create_node_data_dir
10
+ configuration.before_join_hook.call(node.data_dir)
11
+ mark_as_joining
12
+ download_from_slave_nodes
13
+ configuration.after_download_hook.call(BASE_DATA_DIR)
14
+ configuration.after_join_hook.call(node.data_dir)
15
+ end
16
+
17
+ private
18
+
19
+ def download_from_slave_nodes
20
+ # TODO: Consider implementing timeout mechanism
21
+ Parallel.puts('Waiting for slave nodes to be ready for download...')
22
+ loop do
23
+ downloaders.each(&:download)
24
+ break if downloaders.all?(&:downloaded?)
25
+ Kernel.sleep(1)
26
+ end
27
+ end
28
+
29
+ def downloaders
30
+ @downloaders ||= node.other_nodes.map { |other_node| Downloader.new(other_node) }
31
+ end
32
+
33
+ Downloader = Struct.new(:node) do
34
+ def ready_for_download?
35
+ Kernel.system('ssh', node.ssh_host, 'test', '-f', JOIN_MARKER_FILE)
36
+ end
37
+
38
+ def download
39
+ return if downloaded?
40
+ return unless ready_for_download?
41
+ Parallel.puts("Downloading data from #{node.ssh_host}...")
42
+ @downloaded = scp
43
+ mark_as_downloaded if downloaded?
44
+ end
45
+
46
+ def scp
47
+ Kernel.system('scp', '-q', '-r', "#{node.ssh_host}:#{node.data_dir}", BASE_DATA_DIR)
48
+ end
49
+
50
+ def downloaded?
51
+ @downloaded
52
+ end
53
+
54
+ def mark_as_downloaded
55
+ Kernel.system('ssh', node.ssh_host, 'touch', DOWNLOAD_MARKER_FILE)
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,30 @@
1
+ require 'circleci/parallel/task/base'
2
+
3
+ module CircleCI
4
+ module Parallel
5
+ module Task
6
+ # @api private
7
+ class Slave < Base
8
+ def run
9
+ create_node_data_dir
10
+ configuration.before_join_hook.call(node.data_dir)
11
+ mark_as_joining
12
+ wait_for_master_node_to_download
13
+ configuration.after_join_hook.call(node.data_dir)
14
+ end
15
+
16
+ private
17
+
18
+ def wait_for_master_node_to_download
19
+ # TODO: Consider implementing timeout mechanism
20
+ Parallel.puts('Waiting for master node to download data...')
21
+ Kernel.sleep(1) until downloaded?
22
+ end
23
+
24
+ def downloaded?
25
+ File.exist?(DOWNLOAD_MARKER_FILE)
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,13 @@
1
+ module CircleCI
2
+ module Parallel
3
+ module Version
4
+ MAJOR = 0
5
+ MINOR = 1
6
+ PATCH = 0
7
+
8
+ def self.to_s
9
+ [MAJOR, MINOR, PATCH].join('.')
10
+ end
11
+ end
12
+ end
13
+ end
metadata ADDED
@@ -0,0 +1,82 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: circleci-parallel
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Yuji Nakayama
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-08-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.12'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.12'
27
+ description: Provides Ruby APIs for joining CircleCI's parallel builds and sharing
28
+ files between the builds
29
+ email:
30
+ - nkymyj@gmail.com
31
+ executables: []
32
+ extensions: []
33
+ extra_rdoc_files: []
34
+ files:
35
+ - ".gitignore"
36
+ - ".rspec"
37
+ - ".rubocop.yml"
38
+ - ".yardopts"
39
+ - Gemfile
40
+ - LICENSE.txt
41
+ - README.md
42
+ - Rakefile
43
+ - bin/console
44
+ - bin/setup
45
+ - circle.yml
46
+ - circleci-parallel.gemspec
47
+ - lib/circleci/parallel.rb
48
+ - lib/circleci/parallel/build.rb
49
+ - lib/circleci/parallel/configuration.rb
50
+ - lib/circleci/parallel/environment.rb
51
+ - lib/circleci/parallel/hook.rb
52
+ - lib/circleci/parallel/node.rb
53
+ - lib/circleci/parallel/task/base.rb
54
+ - lib/circleci/parallel/task/master.rb
55
+ - lib/circleci/parallel/task/slave.rb
56
+ - lib/circleci/parallel/version.rb
57
+ homepage: https://github.com/increments/circleci-parallel
58
+ licenses:
59
+ - MIT
60
+ metadata: {}
61
+ post_install_message:
62
+ rdoc_options: []
63
+ require_paths:
64
+ - lib
65
+ required_ruby_version: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ required_rubygems_version: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ requirements: []
76
+ rubyforge_project:
77
+ rubygems_version: 2.6.6
78
+ signing_key:
79
+ specification_version: 4
80
+ summary: Provides Ruby APIs for joining CircleCI's parallel builds and sharing files
81
+ between the builds
82
+ test_files: []