libis-workflow 2.0.24 → 2.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -1
- data/.gitignore +36 -36
- data/.travis.yml +32 -32
- data/Gemfile +4 -4
- data/LICENSE +20 -20
- data/README.md +380 -380
- data/Rakefile +6 -6
- data/lib/libis/exceptions.rb +6 -6
- data/lib/libis/workflow.rb +41 -41
- data/lib/libis/workflow/action.rb +24 -24
- data/lib/libis/workflow/base/dir_item.rb +13 -13
- data/lib/libis/workflow/base/file_item.rb +80 -80
- data/lib/libis/workflow/base/job.rb +83 -83
- data/lib/libis/workflow/base/logging.rb +66 -66
- data/lib/libis/workflow/base/run.rb +95 -95
- data/lib/libis/workflow/base/work_item.rb +173 -173
- data/lib/libis/workflow/base/workflow.rb +149 -149
- data/lib/libis/workflow/config.rb +22 -22
- data/lib/libis/workflow/dir_item.rb +10 -10
- data/lib/libis/workflow/file_item.rb +15 -15
- data/lib/libis/workflow/job.rb +28 -28
- data/lib/libis/workflow/message_registry.rb +30 -30
- data/lib/libis/workflow/run.rb +34 -34
- data/lib/libis/workflow/status.rb +133 -133
- data/lib/libis/workflow/task.rb +316 -316
- data/lib/libis/workflow/task_group.rb +71 -71
- data/lib/libis/workflow/task_runner.rb +34 -34
- data/lib/libis/workflow/version.rb +5 -5
- data/lib/libis/workflow/work_item.rb +37 -37
- data/lib/libis/workflow/worker.rb +42 -42
- data/lib/libis/workflow/workflow.rb +20 -20
- data/libis-workflow.gemspec +38 -38
- data/spec/items.rb +2 -2
- data/spec/items/test_dir_item.rb +13 -13
- data/spec/items/test_file_item.rb +16 -16
- data/spec/items/test_run.rb +8 -8
- data/spec/spec_helper.rb +8 -8
- data/spec/task_spec.rb +15 -15
- data/spec/tasks/camelize_name.rb +12 -12
- data/spec/tasks/checksum_tester.rb +32 -32
- data/spec/tasks/collect_files.rb +47 -47
- data/spec/workflow_spec.rb +154 -154
- metadata +3 -3
data/Rakefile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
require 'bundler/gem_tasks'
|
2
|
-
require 'rspec/core/rake_task'
|
3
|
-
|
4
|
-
RSpec::Core::RakeTask.new('spec')
|
5
|
-
|
6
|
-
desc 'run tests'
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rspec/core/rake_task'
|
3
|
+
|
4
|
+
RSpec::Core::RakeTask.new('spec')
|
5
|
+
|
6
|
+
desc 'run tests'
|
7
7
|
task :default => :spec
|
data/lib/libis/exceptions.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
module Libis
|
2
|
-
class WorkflowError < ::RuntimeError
|
3
|
-
end
|
4
|
-
class WorkflowAbort < ::RuntimeError
|
5
|
-
end
|
6
|
-
end
|
1
|
+
module Libis
|
2
|
+
class WorkflowError < ::RuntimeError
|
3
|
+
end
|
4
|
+
class WorkflowAbort < ::RuntimeError
|
5
|
+
end
|
6
|
+
end
|
data/lib/libis/workflow.rb
CHANGED
@@ -1,41 +1,41 @@
|
|
1
|
-
require 'libis/exceptions'
|
2
|
-
|
3
|
-
require_relative 'workflow/version'
|
4
|
-
|
5
|
-
module Libis
|
6
|
-
module Workflow
|
7
|
-
|
8
|
-
autoload :MessageRegistry, 'libis/workflow/message_registry'
|
9
|
-
autoload :Config, 'libis/workflow/config'
|
10
|
-
|
11
|
-
module Base
|
12
|
-
autoload :WorkItem, 'libis/workflow/base/work_item'
|
13
|
-
autoload :FileItem, 'libis/workflow/base/file_item'
|
14
|
-
autoload :DirItem, 'libis/workflow/base/dir_item'
|
15
|
-
autoload :Logging, 'libis/workflow/base/logging'
|
16
|
-
autoload :Job, 'libis/workflow/base/job'
|
17
|
-
autoload :Run, 'libis/workflow/base/run'
|
18
|
-
autoload :Workflow, 'libis/workflow/base/workflow'
|
19
|
-
end
|
20
|
-
|
21
|
-
autoload :Status, 'libis/workflow/status'
|
22
|
-
|
23
|
-
autoload :WorkItem, 'libis/workflow/work_item'
|
24
|
-
autoload :FileItem, 'libis/workflow/file_item'
|
25
|
-
autoload :DirItem, 'libis/workflow/dir_item'
|
26
|
-
|
27
|
-
autoload :Workflow, 'libis/workflow/workflow'
|
28
|
-
autoload :Job, 'libis/workflow/job'
|
29
|
-
autoload :Run, 'libis/workflow/run'
|
30
|
-
autoload :Task, 'libis/workflow/task'
|
31
|
-
autoload :TaskGroup, 'libis/workflow/task_group'
|
32
|
-
autoload :TaskRunner, 'libis/workflow/task_runner'
|
33
|
-
|
34
|
-
autoload :Worker, 'libis/workflow/worker'
|
35
|
-
|
36
|
-
def self.configure
|
37
|
-
yield Config.instance
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
1
|
+
require 'libis/exceptions'
|
2
|
+
|
3
|
+
require_relative 'workflow/version'
|
4
|
+
|
5
|
+
module Libis
|
6
|
+
module Workflow
|
7
|
+
|
8
|
+
autoload :MessageRegistry, 'libis/workflow/message_registry'
|
9
|
+
autoload :Config, 'libis/workflow/config'
|
10
|
+
|
11
|
+
module Base
|
12
|
+
autoload :WorkItem, 'libis/workflow/base/work_item'
|
13
|
+
autoload :FileItem, 'libis/workflow/base/file_item'
|
14
|
+
autoload :DirItem, 'libis/workflow/base/dir_item'
|
15
|
+
autoload :Logging, 'libis/workflow/base/logging'
|
16
|
+
autoload :Job, 'libis/workflow/base/job'
|
17
|
+
autoload :Run, 'libis/workflow/base/run'
|
18
|
+
autoload :Workflow, 'libis/workflow/base/workflow'
|
19
|
+
end
|
20
|
+
|
21
|
+
autoload :Status, 'libis/workflow/status'
|
22
|
+
|
23
|
+
autoload :WorkItem, 'libis/workflow/work_item'
|
24
|
+
autoload :FileItem, 'libis/workflow/file_item'
|
25
|
+
autoload :DirItem, 'libis/workflow/dir_item'
|
26
|
+
|
27
|
+
autoload :Workflow, 'libis/workflow/workflow'
|
28
|
+
autoload :Job, 'libis/workflow/job'
|
29
|
+
autoload :Run, 'libis/workflow/run'
|
30
|
+
autoload :Task, 'libis/workflow/task'
|
31
|
+
autoload :TaskGroup, 'libis/workflow/task_group'
|
32
|
+
autoload :TaskRunner, 'libis/workflow/task_runner'
|
33
|
+
|
34
|
+
autoload :Worker, 'libis/workflow/worker'
|
35
|
+
|
36
|
+
def self.configure
|
37
|
+
yield Config.instance
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -1,24 +1,24 @@
|
|
1
|
-
module Libis
|
2
|
-
module Workflow
|
3
|
-
module Action
|
4
|
-
|
5
|
-
RUN = 0
|
6
|
-
CONTINUE = 1
|
7
|
-
RETRY = 2
|
8
|
-
ALWAYS_RUN = 3
|
9
|
-
FAILED = 4
|
10
|
-
|
11
|
-
def next_success_action(action)
|
12
|
-
case action
|
13
|
-
when :run, :continue, :retry, :always_run
|
14
|
-
:always_run
|
15
|
-
when :failed
|
16
|
-
:failed
|
17
|
-
else
|
18
|
-
:failed
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
1
|
+
module Libis
|
2
|
+
module Workflow
|
3
|
+
module Action
|
4
|
+
|
5
|
+
RUN = 0
|
6
|
+
CONTINUE = 1
|
7
|
+
RETRY = 2
|
8
|
+
ALWAYS_RUN = 3
|
9
|
+
FAILED = 4
|
10
|
+
|
11
|
+
def next_success_action(action)
|
12
|
+
case action
|
13
|
+
when :run, :continue, :retry, :always_run
|
14
|
+
:always_run
|
15
|
+
when :failed
|
16
|
+
:failed
|
17
|
+
else
|
18
|
+
:failed
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -1,13 +1,13 @@
|
|
1
|
-
require 'libis/workflow/base/file_item'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
module Base
|
6
|
-
|
7
|
-
module DirItem
|
8
|
-
include ::Libis::Workflow::Base::FileItem
|
9
|
-
end
|
10
|
-
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
1
|
+
require 'libis/workflow/base/file_item'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
module Base
|
6
|
+
|
7
|
+
module DirItem
|
8
|
+
include ::Libis::Workflow::Base::FileItem
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -1,80 +1,80 @@
|
|
1
|
-
require 'digest'
|
2
|
-
|
3
|
-
require 'libis/workflow/base/work_item'
|
4
|
-
|
5
|
-
module Libis
|
6
|
-
module Workflow
|
7
|
-
module Base
|
8
|
-
|
9
|
-
# noinspection RubyResolve
|
10
|
-
module FileItem
|
11
|
-
include Libis::Workflow::Base::WorkItem
|
12
|
-
|
13
|
-
def filename
|
14
|
-
File.basename(self.properties['filename']) || self.properties['link']
|
15
|
-
end
|
16
|
-
|
17
|
-
def name
|
18
|
-
self.properties['name'] || self.filename
|
19
|
-
end
|
20
|
-
|
21
|
-
def filelist
|
22
|
-
(self.parent.filelist rescue Array.new).push(filename).compact
|
23
|
-
end
|
24
|
-
|
25
|
-
def filepath
|
26
|
-
self.filelist.join('/')
|
27
|
-
end
|
28
|
-
|
29
|
-
def fullpath
|
30
|
-
self.properties['filename']
|
31
|
-
end
|
32
|
-
|
33
|
-
def filename=(name)
|
34
|
-
begin
|
35
|
-
stats = ::File.stat name
|
36
|
-
self.properties['size'] = stats.size
|
37
|
-
self.properties['access_time'] = stats.atime
|
38
|
-
self.properties['modification_time'] = stats.mtime
|
39
|
-
self.properties['creation_time'] = stats.ctime
|
40
|
-
self.properties['mode'] = stats.mode
|
41
|
-
self.properties['uid'] = stats.uid
|
42
|
-
self.properties['gid'] = stats.gid
|
43
|
-
set_checksum(:MD5, ::Digest::MD5.hexdigest(File.read(name))) if File.file?(name)
|
44
|
-
rescue
|
45
|
-
# ignored
|
46
|
-
end
|
47
|
-
self.properties['filename'] = name
|
48
|
-
end
|
49
|
-
|
50
|
-
def checksum(checksum_type)
|
51
|
-
self.properties[('checksum_' + checksum_type.to_s.downcase)]
|
52
|
-
end
|
53
|
-
|
54
|
-
def set_checksum(checksum_type, value)
|
55
|
-
self.properties[('checksum_' + checksum_type.to_s.downcase)] = value
|
56
|
-
end
|
57
|
-
|
58
|
-
def link
|
59
|
-
self.properties['link']
|
60
|
-
end
|
61
|
-
|
62
|
-
def link=(name)
|
63
|
-
self.properties['link'] = name
|
64
|
-
end
|
65
|
-
|
66
|
-
def set_info(info)
|
67
|
-
info.each do |k, v|
|
68
|
-
self.properties[k] = v
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def safe_name
|
73
|
-
self.name.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
1
|
+
require 'digest'
|
2
|
+
|
3
|
+
require 'libis/workflow/base/work_item'
|
4
|
+
|
5
|
+
module Libis
|
6
|
+
module Workflow
|
7
|
+
module Base
|
8
|
+
|
9
|
+
# noinspection RubyResolve
|
10
|
+
module FileItem
|
11
|
+
include Libis::Workflow::Base::WorkItem
|
12
|
+
|
13
|
+
def filename
|
14
|
+
File.basename(self.properties['filename']) || self.properties['link']
|
15
|
+
end
|
16
|
+
|
17
|
+
def name
|
18
|
+
self.properties['name'] || self.filename
|
19
|
+
end
|
20
|
+
|
21
|
+
def filelist
|
22
|
+
(self.parent.filelist rescue Array.new).push(filename).compact
|
23
|
+
end
|
24
|
+
|
25
|
+
def filepath
|
26
|
+
self.filelist.join('/')
|
27
|
+
end
|
28
|
+
|
29
|
+
def fullpath
|
30
|
+
self.properties['filename']
|
31
|
+
end
|
32
|
+
|
33
|
+
def filename=(name)
|
34
|
+
begin
|
35
|
+
stats = ::File.stat name
|
36
|
+
self.properties['size'] = stats.size
|
37
|
+
self.properties['access_time'] = stats.atime
|
38
|
+
self.properties['modification_time'] = stats.mtime
|
39
|
+
self.properties['creation_time'] = stats.ctime
|
40
|
+
self.properties['mode'] = stats.mode
|
41
|
+
self.properties['uid'] = stats.uid
|
42
|
+
self.properties['gid'] = stats.gid
|
43
|
+
set_checksum(:MD5, ::Digest::MD5.hexdigest(File.read(name))) if File.file?(name)
|
44
|
+
rescue
|
45
|
+
# ignored
|
46
|
+
end
|
47
|
+
self.properties['filename'] = name
|
48
|
+
end
|
49
|
+
|
50
|
+
def checksum(checksum_type)
|
51
|
+
self.properties[('checksum_' + checksum_type.to_s.downcase)]
|
52
|
+
end
|
53
|
+
|
54
|
+
def set_checksum(checksum_type, value)
|
55
|
+
self.properties[('checksum_' + checksum_type.to_s.downcase)] = value
|
56
|
+
end
|
57
|
+
|
58
|
+
def link
|
59
|
+
self.properties['link']
|
60
|
+
end
|
61
|
+
|
62
|
+
def link=(name)
|
63
|
+
self.properties['link'] = name
|
64
|
+
end
|
65
|
+
|
66
|
+
def set_info(info)
|
67
|
+
info.each do |k, v|
|
68
|
+
self.properties[k] = v
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def safe_name
|
73
|
+
self.name.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -1,83 +1,83 @@
|
|
1
|
-
require 'libis/tools/parameter'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
module Base
|
6
|
-
|
7
|
-
# This is the base module for Jobs.
|
8
|
-
#
|
9
|
-
# This module lacks the implementation for the data attributes. It functions as an interface that describes the
|
10
|
-
# common functionality regardless of the storage implementation. These attributes require some implementation:
|
11
|
-
#
|
12
|
-
# - name: [String] the name of the Job. The name will be used to identify the job. Each time a job is executed,
|
13
|
-
# a Run will be created for the associated workflow. The Run will get a name that starts with the job name
|
14
|
-
# and ends with the date and time the Run was first started. As such this name attribute serves as an
|
15
|
-
# identifier and should be treated as such. If possible it should be unique.
|
16
|
-
# - description: [String] optional information about the job.
|
17
|
-
# - workflow: [Object] the workflow containing the tasks that need to run.
|
18
|
-
# - run_obj: [String] the full class name of the Run implementation object that should be created when the
|
19
|
-
# Job is executed.
|
20
|
-
# - input: [Hash] workflow input parameter values. Each input parameter of the workflow can be set by the entries
|
21
|
-
# in this Hash.
|
22
|
-
#
|
23
|
-
# A minimal in-memory implementation could be:
|
24
|
-
#
|
25
|
-
# class Job
|
26
|
-
# include ::Libis::Workflow::Base::Job
|
27
|
-
#
|
28
|
-
# attr_accessor :name, :description, :workflow, :run_object, :input
|
29
|
-
#
|
30
|
-
# def initialize
|
31
|
-
# @name = ''
|
32
|
-
# @description = ''
|
33
|
-
# @input = Hash.new
|
34
|
-
# @workflow = ::Libis::Workflow::Workflow.new
|
35
|
-
# @run_object = ::Libis::Workflow::Run.new
|
36
|
-
# end
|
37
|
-
#
|
38
|
-
# end
|
39
|
-
#
|
40
|
-
module Job
|
41
|
-
|
42
|
-
def run_name(timestamp = Time.now)
|
43
|
-
"#{self.name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
|
44
|
-
end
|
45
|
-
|
46
|
-
def configure(cfg = {})
|
47
|
-
self.name ||= ''
|
48
|
-
self.description ||= ''
|
49
|
-
self.input ||= {}
|
50
|
-
self.name = cfg['name'] if cfg.has_key?('name')
|
51
|
-
self.description = cfg['description'] if cfg.has_key?('description')
|
52
|
-
self.workflow = cfg['workflow'] if cfg.has_key?('workflow')
|
53
|
-
self.run_object = cfg['run_object'] if cfg.has_key?('run_object')
|
54
|
-
self.input.merge!(cfg['input'] || {})
|
55
|
-
end
|
56
|
-
|
57
|
-
# noinspection RubyResolve
|
58
|
-
# @param [Hash] opts optional extra conguration values for this particular run
|
59
|
-
def execute(opts = {})
|
60
|
-
run = self.create_run_object
|
61
|
-
raise RuntimeError.new "Could not create instance of run object '#{self.run_object}'" unless run
|
62
|
-
|
63
|
-
run.job = self
|
64
|
-
(opts.delete('run_config') || {}).each { |key,value| run.send("#{key}=", value) }
|
65
|
-
run.options = self.input.merge(opts)
|
66
|
-
run.save!
|
67
|
-
|
68
|
-
run.run
|
69
|
-
|
70
|
-
run
|
71
|
-
end
|
72
|
-
|
73
|
-
protected
|
74
|
-
|
75
|
-
# noinspection RubyResolve
|
76
|
-
def create_run_object
|
77
|
-
self.run_object.constantize.new
|
78
|
-
end
|
79
|
-
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
1
|
+
require 'libis/tools/parameter'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
module Base
|
6
|
+
|
7
|
+
# This is the base module for Jobs.
|
8
|
+
#
|
9
|
+
# This module lacks the implementation for the data attributes. It functions as an interface that describes the
|
10
|
+
# common functionality regardless of the storage implementation. These attributes require some implementation:
|
11
|
+
#
|
12
|
+
# - name: [String] the name of the Job. The name will be used to identify the job. Each time a job is executed,
|
13
|
+
# a Run will be created for the associated workflow. The Run will get a name that starts with the job name
|
14
|
+
# and ends with the date and time the Run was first started. As such this name attribute serves as an
|
15
|
+
# identifier and should be treated as such. If possible it should be unique.
|
16
|
+
# - description: [String] optional information about the job.
|
17
|
+
# - workflow: [Object] the workflow containing the tasks that need to run.
|
18
|
+
# - run_obj: [String] the full class name of the Run implementation object that should be created when the
|
19
|
+
# Job is executed.
|
20
|
+
# - input: [Hash] workflow input parameter values. Each input parameter of the workflow can be set by the entries
|
21
|
+
# in this Hash.
|
22
|
+
#
|
23
|
+
# A minimal in-memory implementation could be:
|
24
|
+
#
|
25
|
+
# class Job
|
26
|
+
# include ::Libis::Workflow::Base::Job
|
27
|
+
#
|
28
|
+
# attr_accessor :name, :description, :workflow, :run_object, :input
|
29
|
+
#
|
30
|
+
# def initialize
|
31
|
+
# @name = ''
|
32
|
+
# @description = ''
|
33
|
+
# @input = Hash.new
|
34
|
+
# @workflow = ::Libis::Workflow::Workflow.new
|
35
|
+
# @run_object = ::Libis::Workflow::Run.new
|
36
|
+
# end
|
37
|
+
#
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
module Job
|
41
|
+
|
42
|
+
def run_name(timestamp = Time.now)
|
43
|
+
"#{self.name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
|
44
|
+
end
|
45
|
+
|
46
|
+
def configure(cfg = {})
|
47
|
+
self.name ||= ''
|
48
|
+
self.description ||= ''
|
49
|
+
self.input ||= {}
|
50
|
+
self.name = cfg['name'] if cfg.has_key?('name')
|
51
|
+
self.description = cfg['description'] if cfg.has_key?('description')
|
52
|
+
self.workflow = cfg['workflow'] if cfg.has_key?('workflow')
|
53
|
+
self.run_object = cfg['run_object'] if cfg.has_key?('run_object')
|
54
|
+
self.input.merge!(cfg['input'] || {})
|
55
|
+
end
|
56
|
+
|
57
|
+
# noinspection RubyResolve
|
58
|
+
# @param [Hash] opts optional extra conguration values for this particular run
|
59
|
+
def execute(opts = {})
|
60
|
+
run = self.create_run_object
|
61
|
+
raise RuntimeError.new "Could not create instance of run object '#{self.run_object}'" unless run
|
62
|
+
|
63
|
+
run.job = self
|
64
|
+
(opts.delete('run_config') || {}).each { |key,value| run.send("#{key}=", value) }
|
65
|
+
run.options = self.input.merge(opts)
|
66
|
+
run.save!
|
67
|
+
|
68
|
+
run.run
|
69
|
+
|
70
|
+
run
|
71
|
+
end
|
72
|
+
|
73
|
+
protected
|
74
|
+
|
75
|
+
# noinspection RubyResolve
|
76
|
+
def create_run_object
|
77
|
+
self.run_object.constantize.new
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|