libis-workflow 2.0.24 → 2.0.25
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -1
- data/.gitignore +36 -36
- data/.travis.yml +32 -32
- data/Gemfile +4 -4
- data/LICENSE +20 -20
- data/README.md +380 -380
- data/Rakefile +6 -6
- data/lib/libis/exceptions.rb +6 -6
- data/lib/libis/workflow.rb +41 -41
- data/lib/libis/workflow/action.rb +24 -24
- data/lib/libis/workflow/base/dir_item.rb +13 -13
- data/lib/libis/workflow/base/file_item.rb +80 -80
- data/lib/libis/workflow/base/job.rb +83 -83
- data/lib/libis/workflow/base/logging.rb +66 -66
- data/lib/libis/workflow/base/run.rb +95 -95
- data/lib/libis/workflow/base/work_item.rb +173 -173
- data/lib/libis/workflow/base/workflow.rb +149 -149
- data/lib/libis/workflow/config.rb +22 -22
- data/lib/libis/workflow/dir_item.rb +10 -10
- data/lib/libis/workflow/file_item.rb +15 -15
- data/lib/libis/workflow/job.rb +28 -28
- data/lib/libis/workflow/message_registry.rb +30 -30
- data/lib/libis/workflow/run.rb +34 -34
- data/lib/libis/workflow/status.rb +133 -133
- data/lib/libis/workflow/task.rb +316 -316
- data/lib/libis/workflow/task_group.rb +71 -71
- data/lib/libis/workflow/task_runner.rb +34 -34
- data/lib/libis/workflow/version.rb +5 -5
- data/lib/libis/workflow/work_item.rb +37 -37
- data/lib/libis/workflow/worker.rb +42 -42
- data/lib/libis/workflow/workflow.rb +20 -20
- data/libis-workflow.gemspec +38 -38
- data/spec/items.rb +2 -2
- data/spec/items/test_dir_item.rb +13 -13
- data/spec/items/test_file_item.rb +16 -16
- data/spec/items/test_run.rb +8 -8
- data/spec/spec_helper.rb +8 -8
- data/spec/task_spec.rb +15 -15
- data/spec/tasks/camelize_name.rb +12 -12
- data/spec/tasks/checksum_tester.rb +32 -32
- data/spec/tasks/collect_files.rb +47 -47
- data/spec/workflow_spec.rb +154 -154
- metadata +3 -3
data/Rakefile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
require 'bundler/gem_tasks'
|
2
|
-
require 'rspec/core/rake_task'
|
3
|
-
|
4
|
-
RSpec::Core::RakeTask.new('spec')
|
5
|
-
|
6
|
-
desc 'run tests'
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rspec/core/rake_task'
|
3
|
+
|
4
|
+
RSpec::Core::RakeTask.new('spec')
|
5
|
+
|
6
|
+
desc 'run tests'
|
7
7
|
task :default => :spec
|
data/lib/libis/exceptions.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
module Libis
|
2
|
-
class WorkflowError < ::RuntimeError
|
3
|
-
end
|
4
|
-
class WorkflowAbort < ::RuntimeError
|
5
|
-
end
|
6
|
-
end
|
1
|
+
module Libis
|
2
|
+
class WorkflowError < ::RuntimeError
|
3
|
+
end
|
4
|
+
class WorkflowAbort < ::RuntimeError
|
5
|
+
end
|
6
|
+
end
|
data/lib/libis/workflow.rb
CHANGED
@@ -1,41 +1,41 @@
|
|
1
|
-
require 'libis/exceptions'
|
2
|
-
|
3
|
-
require_relative 'workflow/version'
|
4
|
-
|
5
|
-
module Libis
|
6
|
-
module Workflow
|
7
|
-
|
8
|
-
autoload :MessageRegistry, 'libis/workflow/message_registry'
|
9
|
-
autoload :Config, 'libis/workflow/config'
|
10
|
-
|
11
|
-
module Base
|
12
|
-
autoload :WorkItem, 'libis/workflow/base/work_item'
|
13
|
-
autoload :FileItem, 'libis/workflow/base/file_item'
|
14
|
-
autoload :DirItem, 'libis/workflow/base/dir_item'
|
15
|
-
autoload :Logging, 'libis/workflow/base/logging'
|
16
|
-
autoload :Job, 'libis/workflow/base/job'
|
17
|
-
autoload :Run, 'libis/workflow/base/run'
|
18
|
-
autoload :Workflow, 'libis/workflow/base/workflow'
|
19
|
-
end
|
20
|
-
|
21
|
-
autoload :Status, 'libis/workflow/status'
|
22
|
-
|
23
|
-
autoload :WorkItem, 'libis/workflow/work_item'
|
24
|
-
autoload :FileItem, 'libis/workflow/file_item'
|
25
|
-
autoload :DirItem, 'libis/workflow/dir_item'
|
26
|
-
|
27
|
-
autoload :Workflow, 'libis/workflow/workflow'
|
28
|
-
autoload :Job, 'libis/workflow/job'
|
29
|
-
autoload :Run, 'libis/workflow/run'
|
30
|
-
autoload :Task, 'libis/workflow/task'
|
31
|
-
autoload :TaskGroup, 'libis/workflow/task_group'
|
32
|
-
autoload :TaskRunner, 'libis/workflow/task_runner'
|
33
|
-
|
34
|
-
autoload :Worker, 'libis/workflow/worker'
|
35
|
-
|
36
|
-
def self.configure
|
37
|
-
yield Config.instance
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
1
|
+
require 'libis/exceptions'
|
2
|
+
|
3
|
+
require_relative 'workflow/version'
|
4
|
+
|
5
|
+
module Libis
|
6
|
+
module Workflow
|
7
|
+
|
8
|
+
autoload :MessageRegistry, 'libis/workflow/message_registry'
|
9
|
+
autoload :Config, 'libis/workflow/config'
|
10
|
+
|
11
|
+
module Base
|
12
|
+
autoload :WorkItem, 'libis/workflow/base/work_item'
|
13
|
+
autoload :FileItem, 'libis/workflow/base/file_item'
|
14
|
+
autoload :DirItem, 'libis/workflow/base/dir_item'
|
15
|
+
autoload :Logging, 'libis/workflow/base/logging'
|
16
|
+
autoload :Job, 'libis/workflow/base/job'
|
17
|
+
autoload :Run, 'libis/workflow/base/run'
|
18
|
+
autoload :Workflow, 'libis/workflow/base/workflow'
|
19
|
+
end
|
20
|
+
|
21
|
+
autoload :Status, 'libis/workflow/status'
|
22
|
+
|
23
|
+
autoload :WorkItem, 'libis/workflow/work_item'
|
24
|
+
autoload :FileItem, 'libis/workflow/file_item'
|
25
|
+
autoload :DirItem, 'libis/workflow/dir_item'
|
26
|
+
|
27
|
+
autoload :Workflow, 'libis/workflow/workflow'
|
28
|
+
autoload :Job, 'libis/workflow/job'
|
29
|
+
autoload :Run, 'libis/workflow/run'
|
30
|
+
autoload :Task, 'libis/workflow/task'
|
31
|
+
autoload :TaskGroup, 'libis/workflow/task_group'
|
32
|
+
autoload :TaskRunner, 'libis/workflow/task_runner'
|
33
|
+
|
34
|
+
autoload :Worker, 'libis/workflow/worker'
|
35
|
+
|
36
|
+
def self.configure
|
37
|
+
yield Config.instance
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -1,24 +1,24 @@
|
|
1
|
-
module Libis
|
2
|
-
module Workflow
|
3
|
-
module Action
|
4
|
-
|
5
|
-
RUN = 0
|
6
|
-
CONTINUE = 1
|
7
|
-
RETRY = 2
|
8
|
-
ALWAYS_RUN = 3
|
9
|
-
FAILED = 4
|
10
|
-
|
11
|
-
def next_success_action(action)
|
12
|
-
case action
|
13
|
-
when :run, :continue, :retry, :always_run
|
14
|
-
:always_run
|
15
|
-
when :failed
|
16
|
-
:failed
|
17
|
-
else
|
18
|
-
:failed
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
1
|
+
module Libis
|
2
|
+
module Workflow
|
3
|
+
module Action
|
4
|
+
|
5
|
+
RUN = 0
|
6
|
+
CONTINUE = 1
|
7
|
+
RETRY = 2
|
8
|
+
ALWAYS_RUN = 3
|
9
|
+
FAILED = 4
|
10
|
+
|
11
|
+
def next_success_action(action)
|
12
|
+
case action
|
13
|
+
when :run, :continue, :retry, :always_run
|
14
|
+
:always_run
|
15
|
+
when :failed
|
16
|
+
:failed
|
17
|
+
else
|
18
|
+
:failed
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -1,13 +1,13 @@
|
|
1
|
-
require 'libis/workflow/base/file_item'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
module Base
|
6
|
-
|
7
|
-
module DirItem
|
8
|
-
include ::Libis::Workflow::Base::FileItem
|
9
|
-
end
|
10
|
-
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
1
|
+
require 'libis/workflow/base/file_item'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
module Base
|
6
|
+
|
7
|
+
module DirItem
|
8
|
+
include ::Libis::Workflow::Base::FileItem
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -1,80 +1,80 @@
|
|
1
|
-
require 'digest'
|
2
|
-
|
3
|
-
require 'libis/workflow/base/work_item'
|
4
|
-
|
5
|
-
module Libis
|
6
|
-
module Workflow
|
7
|
-
module Base
|
8
|
-
|
9
|
-
# noinspection RubyResolve
|
10
|
-
module FileItem
|
11
|
-
include Libis::Workflow::Base::WorkItem
|
12
|
-
|
13
|
-
def filename
|
14
|
-
File.basename(self.properties['filename']) || self.properties['link']
|
15
|
-
end
|
16
|
-
|
17
|
-
def name
|
18
|
-
self.properties['name'] || self.filename
|
19
|
-
end
|
20
|
-
|
21
|
-
def filelist
|
22
|
-
(self.parent.filelist rescue Array.new).push(filename).compact
|
23
|
-
end
|
24
|
-
|
25
|
-
def filepath
|
26
|
-
self.filelist.join('/')
|
27
|
-
end
|
28
|
-
|
29
|
-
def fullpath
|
30
|
-
self.properties['filename']
|
31
|
-
end
|
32
|
-
|
33
|
-
def filename=(name)
|
34
|
-
begin
|
35
|
-
stats = ::File.stat name
|
36
|
-
self.properties['size'] = stats.size
|
37
|
-
self.properties['access_time'] = stats.atime
|
38
|
-
self.properties['modification_time'] = stats.mtime
|
39
|
-
self.properties['creation_time'] = stats.ctime
|
40
|
-
self.properties['mode'] = stats.mode
|
41
|
-
self.properties['uid'] = stats.uid
|
42
|
-
self.properties['gid'] = stats.gid
|
43
|
-
set_checksum(:MD5, ::Digest::MD5.hexdigest(File.read(name))) if File.file?(name)
|
44
|
-
rescue
|
45
|
-
# ignored
|
46
|
-
end
|
47
|
-
self.properties['filename'] = name
|
48
|
-
end
|
49
|
-
|
50
|
-
def checksum(checksum_type)
|
51
|
-
self.properties[('checksum_' + checksum_type.to_s.downcase)]
|
52
|
-
end
|
53
|
-
|
54
|
-
def set_checksum(checksum_type, value)
|
55
|
-
self.properties[('checksum_' + checksum_type.to_s.downcase)] = value
|
56
|
-
end
|
57
|
-
|
58
|
-
def link
|
59
|
-
self.properties['link']
|
60
|
-
end
|
61
|
-
|
62
|
-
def link=(name)
|
63
|
-
self.properties['link'] = name
|
64
|
-
end
|
65
|
-
|
66
|
-
def set_info(info)
|
67
|
-
info.each do |k, v|
|
68
|
-
self.properties[k] = v
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def safe_name
|
73
|
-
self.name.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
1
|
+
require 'digest'
|
2
|
+
|
3
|
+
require 'libis/workflow/base/work_item'
|
4
|
+
|
5
|
+
module Libis
|
6
|
+
module Workflow
|
7
|
+
module Base
|
8
|
+
|
9
|
+
# noinspection RubyResolve
|
10
|
+
module FileItem
|
11
|
+
include Libis::Workflow::Base::WorkItem
|
12
|
+
|
13
|
+
def filename
|
14
|
+
File.basename(self.properties['filename']) || self.properties['link']
|
15
|
+
end
|
16
|
+
|
17
|
+
def name
|
18
|
+
self.properties['name'] || self.filename
|
19
|
+
end
|
20
|
+
|
21
|
+
def filelist
|
22
|
+
(self.parent.filelist rescue Array.new).push(filename).compact
|
23
|
+
end
|
24
|
+
|
25
|
+
def filepath
|
26
|
+
self.filelist.join('/')
|
27
|
+
end
|
28
|
+
|
29
|
+
def fullpath
|
30
|
+
self.properties['filename']
|
31
|
+
end
|
32
|
+
|
33
|
+
def filename=(name)
|
34
|
+
begin
|
35
|
+
stats = ::File.stat name
|
36
|
+
self.properties['size'] = stats.size
|
37
|
+
self.properties['access_time'] = stats.atime
|
38
|
+
self.properties['modification_time'] = stats.mtime
|
39
|
+
self.properties['creation_time'] = stats.ctime
|
40
|
+
self.properties['mode'] = stats.mode
|
41
|
+
self.properties['uid'] = stats.uid
|
42
|
+
self.properties['gid'] = stats.gid
|
43
|
+
set_checksum(:MD5, ::Digest::MD5.hexdigest(File.read(name))) if File.file?(name)
|
44
|
+
rescue
|
45
|
+
# ignored
|
46
|
+
end
|
47
|
+
self.properties['filename'] = name
|
48
|
+
end
|
49
|
+
|
50
|
+
def checksum(checksum_type)
|
51
|
+
self.properties[('checksum_' + checksum_type.to_s.downcase)]
|
52
|
+
end
|
53
|
+
|
54
|
+
def set_checksum(checksum_type, value)
|
55
|
+
self.properties[('checksum_' + checksum_type.to_s.downcase)] = value
|
56
|
+
end
|
57
|
+
|
58
|
+
def link
|
59
|
+
self.properties['link']
|
60
|
+
end
|
61
|
+
|
62
|
+
def link=(name)
|
63
|
+
self.properties['link'] = name
|
64
|
+
end
|
65
|
+
|
66
|
+
def set_info(info)
|
67
|
+
info.each do |k, v|
|
68
|
+
self.properties[k] = v
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def safe_name
|
73
|
+
self.name.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -1,83 +1,83 @@
|
|
1
|
-
require 'libis/tools/parameter'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
module Base
|
6
|
-
|
7
|
-
# This is the base module for Jobs.
|
8
|
-
#
|
9
|
-
# This module lacks the implementation for the data attributes. It functions as an interface that describes the
|
10
|
-
# common functionality regardless of the storage implementation. These attributes require some implementation:
|
11
|
-
#
|
12
|
-
# - name: [String] the name of the Job. The name will be used to identify the job. Each time a job is executed,
|
13
|
-
# a Run will be created for the associated workflow. The Run will get a name that starts with the job name
|
14
|
-
# and ends with the date and time the Run was first started. As such this name attribute serves as an
|
15
|
-
# identifier and should be treated as such. If possible it should be unique.
|
16
|
-
# - description: [String] optional information about the job.
|
17
|
-
# - workflow: [Object] the workflow containing the tasks that need to run.
|
18
|
-
# - run_obj: [String] the full class name of the Run implementation object that should be created when the
|
19
|
-
# Job is executed.
|
20
|
-
# - input: [Hash] workflow input parameter values. Each input parameter of the workflow can be set by the entries
|
21
|
-
# in this Hash.
|
22
|
-
#
|
23
|
-
# A minimal in-memory implementation could be:
|
24
|
-
#
|
25
|
-
# class Job
|
26
|
-
# include ::Libis::Workflow::Base::Job
|
27
|
-
#
|
28
|
-
# attr_accessor :name, :description, :workflow, :run_object, :input
|
29
|
-
#
|
30
|
-
# def initialize
|
31
|
-
# @name = ''
|
32
|
-
# @description = ''
|
33
|
-
# @input = Hash.new
|
34
|
-
# @workflow = ::Libis::Workflow::Workflow.new
|
35
|
-
# @run_object = ::Libis::Workflow::Run.new
|
36
|
-
# end
|
37
|
-
#
|
38
|
-
# end
|
39
|
-
#
|
40
|
-
module Job
|
41
|
-
|
42
|
-
def run_name(timestamp = Time.now)
|
43
|
-
"#{self.name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
|
44
|
-
end
|
45
|
-
|
46
|
-
def configure(cfg = {})
|
47
|
-
self.name ||= ''
|
48
|
-
self.description ||= ''
|
49
|
-
self.input ||= {}
|
50
|
-
self.name = cfg['name'] if cfg.has_key?('name')
|
51
|
-
self.description = cfg['description'] if cfg.has_key?('description')
|
52
|
-
self.workflow = cfg['workflow'] if cfg.has_key?('workflow')
|
53
|
-
self.run_object = cfg['run_object'] if cfg.has_key?('run_object')
|
54
|
-
self.input.merge!(cfg['input'] || {})
|
55
|
-
end
|
56
|
-
|
57
|
-
# noinspection RubyResolve
|
58
|
-
# @param [Hash] opts optional extra conguration values for this particular run
|
59
|
-
def execute(opts = {})
|
60
|
-
run = self.create_run_object
|
61
|
-
raise RuntimeError.new "Could not create instance of run object '#{self.run_object}'" unless run
|
62
|
-
|
63
|
-
run.job = self
|
64
|
-
(opts.delete('run_config') || {}).each { |key,value| run.send("#{key}=", value) }
|
65
|
-
run.options = self.input.merge(opts)
|
66
|
-
run.save!
|
67
|
-
|
68
|
-
run.run
|
69
|
-
|
70
|
-
run
|
71
|
-
end
|
72
|
-
|
73
|
-
protected
|
74
|
-
|
75
|
-
# noinspection RubyResolve
|
76
|
-
def create_run_object
|
77
|
-
self.run_object.constantize.new
|
78
|
-
end
|
79
|
-
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
1
|
+
require 'libis/tools/parameter'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
module Base
|
6
|
+
|
7
|
+
# This is the base module for Jobs.
|
8
|
+
#
|
9
|
+
# This module lacks the implementation for the data attributes. It functions as an interface that describes the
|
10
|
+
# common functionality regardless of the storage implementation. These attributes require some implementation:
|
11
|
+
#
|
12
|
+
# - name: [String] the name of the Job. The name will be used to identify the job. Each time a job is executed,
|
13
|
+
# a Run will be created for the associated workflow. The Run will get a name that starts with the job name
|
14
|
+
# and ends with the date and time the Run was first started. As such this name attribute serves as an
|
15
|
+
# identifier and should be treated as such. If possible it should be unique.
|
16
|
+
# - description: [String] optional information about the job.
|
17
|
+
# - workflow: [Object] the workflow containing the tasks that need to run.
|
18
|
+
# - run_obj: [String] the full class name of the Run implementation object that should be created when the
|
19
|
+
# Job is executed.
|
20
|
+
# - input: [Hash] workflow input parameter values. Each input parameter of the workflow can be set by the entries
|
21
|
+
# in this Hash.
|
22
|
+
#
|
23
|
+
# A minimal in-memory implementation could be:
|
24
|
+
#
|
25
|
+
# class Job
|
26
|
+
# include ::Libis::Workflow::Base::Job
|
27
|
+
#
|
28
|
+
# attr_accessor :name, :description, :workflow, :run_object, :input
|
29
|
+
#
|
30
|
+
# def initialize
|
31
|
+
# @name = ''
|
32
|
+
# @description = ''
|
33
|
+
# @input = Hash.new
|
34
|
+
# @workflow = ::Libis::Workflow::Workflow.new
|
35
|
+
# @run_object = ::Libis::Workflow::Run.new
|
36
|
+
# end
|
37
|
+
#
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
module Job
|
41
|
+
|
42
|
+
def run_name(timestamp = Time.now)
|
43
|
+
"#{self.name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
|
44
|
+
end
|
45
|
+
|
46
|
+
def configure(cfg = {})
|
47
|
+
self.name ||= ''
|
48
|
+
self.description ||= ''
|
49
|
+
self.input ||= {}
|
50
|
+
self.name = cfg['name'] if cfg.has_key?('name')
|
51
|
+
self.description = cfg['description'] if cfg.has_key?('description')
|
52
|
+
self.workflow = cfg['workflow'] if cfg.has_key?('workflow')
|
53
|
+
self.run_object = cfg['run_object'] if cfg.has_key?('run_object')
|
54
|
+
self.input.merge!(cfg['input'] || {})
|
55
|
+
end
|
56
|
+
|
57
|
+
# noinspection RubyResolve
|
58
|
+
# @param [Hash] opts optional extra conguration values for this particular run
|
59
|
+
def execute(opts = {})
|
60
|
+
run = self.create_run_object
|
61
|
+
raise RuntimeError.new "Could not create instance of run object '#{self.run_object}'" unless run
|
62
|
+
|
63
|
+
run.job = self
|
64
|
+
(opts.delete('run_config') || {}).each { |key,value| run.send("#{key}=", value) }
|
65
|
+
run.options = self.input.merge(opts)
|
66
|
+
run.save!
|
67
|
+
|
68
|
+
run.run
|
69
|
+
|
70
|
+
run
|
71
|
+
end
|
72
|
+
|
73
|
+
protected
|
74
|
+
|
75
|
+
# noinspection RubyResolve
|
76
|
+
def create_run_object
|
77
|
+
self.run_object.constantize.new
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|