jongleur 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +1 -0
- data/CHANGELOG.md +13 -0
- data/README.md +53 -57
- data/bin/setup +15 -2
- data/exe/test_gem.sh +3 -0
- data/jongleur.gemspec +3 -1
- data/lib/jongleur/version.rb +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07160e415a435e146c2b162340f0bacef4f3fb27
|
4
|
+
data.tar.gz: 8205d5c8ea466a6e9b1e89d4cc03a5d2f2aba548
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9755c27900cda89b1a8c1a083d6fd05cf62f90ec1a5fd1ed796b57a35ef5049acb740984085c19e2e7feeca41f789fa4f67a1daea9574f99eef908e5a810ae66
|
7
|
+
data.tar.gz: 1dfc257b182634e4af25886e64fdc46986258d6739db8b5e27d0682db2b4f4989ef576eaedf3480fb7c86d574681220a3651b912e93ad5d7b3c13b093dad622a
|
data/.gitlab-ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -4,3 +4,16 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
|
5
5
|
## [1.0.0] - 27-Aug-2018
|
6
6
|
### Initial Release.
|
7
|
+
|
8
|
+
## [1.0.1] - 27-Aug-2018
|
9
|
+
### Release management issues.
|
10
|
+
|
11
|
+
## [1.0.2] - 27-Aug-2018
|
12
|
+
### Release management issues.
|
13
|
+
|
14
|
+
## [1.0.3] - 27-Aug-2018
|
15
|
+
### Release management issues.
|
16
|
+
|
17
|
+
## [1.0.4] - 20-Oct-2018
|
18
|
+
### Added
|
19
|
+
- bin/setup enhanced to install Graphviz dependency
|
data/README.md
CHANGED
@@ -135,74 +135,73 @@ Using Jongleur is easy:
|
|
135
135
|
|
136
136
|
2. Define your Task Graph
|
137
137
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
138
|
+
|
139
|
+
test_graph = {
|
140
|
+
A: [:B, :C],
|
141
|
+
B: [:D],
|
142
|
+
C: [:D],
|
143
|
+
D: [:E],
|
144
|
+
E: []
|
145
|
+
}
|
146
|
+
|
147
147
|
|
148
148
|
3. Add your Task Graph to Jongleur
|
149
149
|
|
150
|
-
|
151
|
-
|
150
|
+
|
151
|
+
API.add_task_graph test_graph
|
152
|
+
|
153
|
+
=> [#<struct Jongleur::Task name=:A, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
154
|
+
#<struct Jongleur::Task name=:B, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
155
|
+
#<struct Jongleur::Task name=:C, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
156
|
+
#<struct Jongleur::Task name=:D, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
157
|
+
#<struct Jongleur::Task name=:E, pid=-1, running=false, exit_status=nil, success_status=nil>]
|
158
|
+
|
152
159
|
|
153
|
-
=> [#<struct Jongleur::Task name=:A, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
154
|
-
#<struct Jongleur::Task name=:B, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
155
|
-
#<struct Jongleur::Task name=:C, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
156
|
-
#<struct Jongleur::Task name=:D, pid=-1, running=false, exit_status=nil, success_status=nil>,
|
157
|
-
#<struct Jongleur::Task name=:E, pid=-1, running=false, exit_status=nil, success_status=nil>]
|
158
|
-
```
|
159
160
|
Jongleur will show you the Task Matrix for your Task Graph with all attributes set at their initial values, obviously, since the Tasks haven't ran yet.
|
160
161
|
|
161
162
|
4. (Optional) You may want to see a graphical representation of your Task Graph
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
163
|
+
|
164
|
+
|
165
|
+
API.print_graph('/tmp')
|
166
|
+
|
167
|
+
=> "/tmp/jongleur_graph_08252018_194828.pdf"
|
168
|
+
|
168
169
|
Opening the PDF file will display this:
|
169
170
|
|
170
171
|
<img src="./bin/img/DAG_graph_1.png" width="225" height="450" alt="ETL DAG">
|
171
172
|
|
172
173
|
5. Implement your tasks. To do that you have to (i) create a new class, based on `WorkerTask` and (ii) define and `#execute` method in your class. This is the method hat Jongleur will call to run the Task. For instance task A from your Task Graph may look something like that:
|
173
174
|
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
175
|
+
|
176
|
+
class A < Jongleur::WorkerTask
|
177
|
+
@desc = 'this is task A'
|
178
|
+
def execute
|
179
|
+
sleep 1
|
180
|
+
'A is running... '
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
You'll have to do the same for Tasks B, C, D and E, as these ae the tasks declared in the Task Graph.
|
184
185
|
|
185
186
|
6. Run the tasks. Ok, pay attention now because this is the complex bit. Nah, only joking - it's simply:
|
186
187
|
|
187
|
-
|
188
|
-
|
189
|
-
|
188
|
+
$> API.run
|
189
|
+
|
190
|
+
=> Starting workflow...
|
191
|
+
=> starting task A
|
192
|
+
=> finished task: A, process: 2501, exit_status: 0, success: true
|
193
|
+
=> starting task B
|
194
|
+
=> starting task C
|
195
|
+
=> finished task: C, process: 2503, exit_status: 0, success: true
|
196
|
+
=> finished task: B, process: 2502, exit_status: 0, success: true
|
197
|
+
=> starting task D
|
198
|
+
=> finished task: D, process: 2505, exit_status: 0, success: true
|
199
|
+
=> starting task E
|
200
|
+
=> finished task: E, process: 2506, exit_status: 0, success: true
|
201
|
+
=> Workflow finished
|
202
|
+
|
190
203
|
|
191
|
-
|
192
|
-
=> starting task A
|
193
|
-
=> finished task: A, process: 2501, exit_status: 0, success: true
|
194
|
-
=> starting task B
|
195
|
-
=> starting task C
|
196
|
-
=> finished task: C, process: 2503, exit_status: 0, success: true
|
197
|
-
=> finished task: B, process: 2502, exit_status: 0, success: true
|
198
|
-
=> starting task D
|
199
|
-
=> finished task: D, process: 2505, exit_status: 0, success: true
|
200
|
-
=> starting task E
|
201
|
-
=> finished task: E, process: 2506, exit_status: 0, success: true
|
202
|
-
=> Workflow finished
|
203
|
-
```
|
204
|
-
|
205
|
-
A __simple example__ of a client app fro Jongleur can be found [on GitLab](https://gitlab.com/RedFred7/jongleur-client)
|
204
|
+
A simple example of a client app for Jongleur can be found [on GitLab](https://gitlab.com/RedFred7/jongleur-client)
|
206
205
|
|
207
206
|
## Use-Cases
|
208
207
|
### Extract-Transform-Load
|
@@ -213,7 +212,7 @@ The ETL workflow is ideally suited to Jongleur. You can define many Extraction
|
|
213
212
|
### Transactions
|
214
213
|
Transactional workflows can be greatly sped up by Jongleur by parallelising parts of the transaction that are usually performed sequentially, i.e:
|
215
214
|
|
216
|
-
<img src="./bin/img/transactional_DAG.png" width="550" height="450" alt="Transaction DAG">
|
215
|
+
<img src="./bin/img/transactional_DAG.png" width="550" height="450" alt="Transaction DAG">
|
217
216
|
|
218
217
|
## Development
|
219
218
|
|
@@ -248,17 +247,14 @@ If a Task fails to run or to finish its run, Jongleur will simply go on running
|
|
248
247
|
### How can I examine the Task Matrix after Jongleur has finished?
|
249
248
|
Jongleur serializes each run's Task Matrix as a JSON file in the `/tmp` directory. You can either view this in an editor or load it and manipulate it in Ruby with
|
250
249
|
|
251
|
-
|
252
|
-
JSON.parse( File.read('/tmp/jongleur_task_matrix_08272018_103406.json') )
|
253
|
-
```
|
250
|
+
`JSON.parse( File.read('/tmp/jongleur_task_matrix_08272018_103406.json') )`
|
254
251
|
|
255
252
|
|
256
253
|
## Roadmap
|
257
254
|
|
258
255
|
These are the things I'd like Jongleur to support in future releases:
|
259
256
|
|
260
|
-
* Task storage mechanism, i.e. the ability for each Task to save data in a uniquely identifiable and safe way so that data can be shared between
|
261
|
-
sequential tasks in a transparent and easy manner.
|
257
|
+
* Task storage mechanism, i.e. the ability for each Task to save data in a uniquely identifiable and safe way so that data can be shared between sequential tasks in a transparent and easy manner.
|
262
258
|
* Rails integration. Pretty self-explanatory really.
|
263
259
|
|
264
260
|
## Contributing
|
data/bin/setup
CHANGED
@@ -3,6 +3,19 @@ set -euo pipefail
|
|
3
3
|
IFS=$'\n\t'
|
4
4
|
set -vx
|
5
5
|
|
6
|
-
|
6
|
+
unameOut="$(uname -s)"
|
7
|
+
case "${unameOut}" in
|
8
|
+
Linux*) machine=Linux;;
|
9
|
+
Darwin*) machine=Mac;;
|
10
|
+
CYGWIN*) machine=Cygwin;;
|
11
|
+
MINGW*) machine=MinGw;;
|
12
|
+
*) machine="UNKNOWN:${unameOut}"
|
13
|
+
esac
|
14
|
+
|
15
|
+
if [[ $machine == 'Linux' ]]; then
|
16
|
+
sudo apt-get install graphviz
|
17
|
+
elif [[ $machine == 'Mac' ]]; then
|
18
|
+
brew cask install graphviz
|
19
|
+
fi
|
7
20
|
|
8
|
-
|
21
|
+
bundle install
|
data/exe/test_gem.sh
ADDED
data/jongleur.gemspec
CHANGED
@@ -28,7 +28,9 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.add_development_dependency 'pry-byebug', '~> 3.4'
|
29
29
|
spec.add_development_dependency 'rake', '~> 10.0'
|
30
30
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
31
|
-
spec.add_development_dependency 'rubocop', '~> 0.58'
|
31
|
+
spec.add_development_dependency 'rubocop', '~> 0.58'
|
32
32
|
spec.add_development_dependency 'simplecov', '~> 0.9'
|
33
33
|
spec.add_development_dependency 'yard', '~> 0.9'
|
34
|
+
|
35
|
+
spec.post_install_message = 'Before using Jongleur, please make sure your system has the Graphviz package installed!'
|
34
36
|
end
|
data/lib/jongleur/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jongleur
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Fred Heath
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-10-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: graphviz
|
@@ -140,7 +140,8 @@ description: Launches, schedules and manages tasks represented in a DAG as multi
|
|
140
140
|
processes
|
141
141
|
email:
|
142
142
|
- fred@bootstrap.me.uk
|
143
|
-
executables:
|
143
|
+
executables:
|
144
|
+
- test_gem.sh
|
144
145
|
extensions: []
|
145
146
|
extra_rdoc_files: []
|
146
147
|
files:
|
@@ -159,6 +160,7 @@ files:
|
|
159
160
|
- bin/img/jongleur_m-2015.jpg
|
160
161
|
- bin/img/transactional_DAG.png
|
161
162
|
- bin/setup
|
163
|
+
- exe/test_gem.sh
|
162
164
|
- jongleur.gemspec
|
163
165
|
- lib/jongleur.rb
|
164
166
|
- lib/jongleur/api.rb
|
@@ -170,7 +172,8 @@ homepage: https://gitlab.com/RedFred7/Jongleur
|
|
170
172
|
licenses:
|
171
173
|
- MIT
|
172
174
|
metadata: {}
|
173
|
-
post_install_message:
|
175
|
+
post_install_message: Before using Jongleur, please make sure your system has the
|
176
|
+
Graphviz package installed!
|
174
177
|
rdoc_options: []
|
175
178
|
require_paths:
|
176
179
|
- lib
|