dipa 0.1.0.pre.1 → 0.1.0.pre.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +77 -49
- data/Rakefile +50 -0
- data/app/models/concerns/dipa/models/agent_state_attributes.rb +24 -0
- data/app/models/concerns/dipa/models/coordinator_state_attributes.rb +24 -0
- data/app/models/concerns/{models/dipa → dipa/models}/dumpable.rb +3 -3
- data/app/models/concerns/{models/dipa → dipa/models}/loadable.rb +4 -4
- data/app/models/concerns/dipa/models/setting_constants.rb +23 -0
- data/app/models/concerns/dipa/models/state_attribute_handling.rb +77 -0
- data/app/models/dipa/agent.rb +14 -13
- data/app/models/dipa/coordinator.rb +32 -19
- data/app/models/dipa/coordinator_options_record.rb +52 -0
- data/app/services/concerns/dipa/services/process_failure_handling.rb +80 -0
- data/app/services/dipa/agent_services/base_service.rb +21 -0
- data/app/services/dipa/agent_services/coordinator_state_service.rb +58 -4
- data/app/services/dipa/agent_services/post_processing_service.rb +5 -3
- data/app/services/dipa/agent_services/processing_service.rb +84 -3
- data/app/services/dipa/agent_services/start_processing_service.rb +6 -4
- data/app/services/dipa/coordinator_services/base_service.rb +21 -0
- data/app/services/dipa/coordinator_services/create_agents_service.rb +7 -5
- data/app/services/dipa/coordinator_services/destroy_service.rb +13 -0
- data/app/services/dipa/coordinator_services/maybe_cleanup_service.rb +20 -0
- data/app/services/dipa/coordinator_services/start_processing_service.rb +6 -4
- data/db/migrate/20220102132652_create_dipa_coordinators.rb +3 -5
- data/db/migrate/20220106183616_create_dipa_agents.rb +3 -3
- data/db/migrate/20230624053724_create_dipa_coordinator_options_record.rb +21 -0
- data/lib/dipa/engine.rb +15 -8
- data/lib/dipa/errors.rb +20 -3
- data/lib/dipa/processor/base.rb +20 -106
- data/lib/dipa/processor/concerns/coordinator.rb +86 -0
- data/lib/dipa/processor/concerns/options.rb +80 -0
- data/lib/dipa/processor/concerns/source.rb +17 -0
- data/lib/dipa/processor/concerns/state.rb +27 -0
- data/lib/dipa/processor/concerns/wait.rb +62 -0
- data/lib/dipa/processor/map.rb +1 -2
- data/lib/dipa/version.rb +3 -1
- data/lib/dipa.rb +18 -0
- metadata +54 -25
- data/app/models/concerns/models/dipa/state_attribute_handling.rb +0 -38
- data/app/models/modules/models/dipa/status_constants.rb +0 -24
- data/lib/tasks/auto_annotate_models.rake +0 -62
data/lib/dipa/processor/base.rb
CHANGED
@@ -3,94 +3,46 @@
|
|
3
3
|
module Dipa
|
4
4
|
module Processor
|
5
5
|
class Base
|
6
|
-
SYNC_MODE_WAIT_CYCLE_SECONDS = 2
|
7
|
-
|
8
|
-
DEFAULT_OPTIONS = {
|
9
|
-
# queue names
|
10
|
-
agent_queue: Dipa.agent_queue,
|
11
|
-
coordinator_queue: Dipa.coordinator_queue,
|
12
|
-
# timeouts
|
13
|
-
agent_timeout: Dipa.agent_timeout,
|
14
|
-
coordinator_timeout: Dipa.coordinator_timeout,
|
15
|
-
# misc
|
16
|
-
async: false,
|
17
|
-
keep_data: false,
|
18
|
-
want_result: true
|
19
|
-
}.freeze
|
20
6
|
OVERRIDE_OPTIONS = {}.freeze
|
21
7
|
|
8
|
+
include Dipa::Processor::Concerns::Coordinator
|
9
|
+
include Dipa::Processor::Concerns::Options
|
10
|
+
include Dipa::Processor::Concerns::Source
|
11
|
+
include Dipa::Processor::Concerns::State
|
12
|
+
include Dipa::Processor::Concerns::Wait
|
13
|
+
|
22
14
|
def with(processor_class, processor_method)
|
23
|
-
|
24
|
-
processor_method: processor_method.to_s)
|
15
|
+
_check_state
|
25
16
|
|
26
17
|
_prepare_coordinator(processor_class: processor_class.to_s,
|
27
18
|
processor_method: processor_method.to_s)
|
28
19
|
|
29
20
|
_start_process
|
30
21
|
|
31
|
-
|
32
|
-
|
33
|
-
|
22
|
+
_result_and_cleanup
|
23
|
+
ensure
|
24
|
+
_reset_state
|
34
25
|
end
|
35
26
|
|
36
27
|
private
|
37
28
|
|
38
|
-
attr_reader :_source, :_raw_opts, :_coordinator
|
39
|
-
|
40
29
|
def initialize(source, options: {})
|
41
|
-
options
|
30
|
+
_validate_options(options: options)
|
31
|
+
_write_options(options: options)
|
42
32
|
|
43
33
|
@_source = source
|
44
|
-
@_raw_opts = options
|
45
|
-
end
|
46
|
-
|
47
|
-
def _validate_processor_arguments(processor_class:, processor_method:)
|
48
|
-
return if processor_class.constantize.respond_to?(processor_method)
|
49
|
-
|
50
|
-
raise Dipa::UnknownProcessorMethodError,
|
51
|
-
"Method .#{processor_method} does not exist on processor class " \
|
52
|
-
"#{processor_class}"
|
53
|
-
rescue NameError => e
|
54
|
-
raise Dipa::UnknownProcessorClassError, e.original_message
|
55
|
-
end
|
56
|
-
|
57
|
-
def _wait_for_it
|
58
|
-
sleep(SYNC_MODE_WAIT_CYCLE_SECONDS) while _wait_for_it?
|
59
|
-
|
60
|
-
return _result if _coordinator.processed?
|
61
|
-
|
62
|
-
# must be an error then
|
63
|
-
_raise_error
|
64
|
-
end
|
65
|
-
|
66
|
-
def _wait_for_it?
|
67
|
-
_coordinator.reload
|
68
|
-
|
69
|
-
_coordinator.initialized? || _coordinator.processing?
|
70
|
-
end
|
71
|
-
|
72
|
-
def _raise_error
|
73
|
-
raise Dipa::AbortedError if _coordinator.aborted?
|
74
|
-
raise Dipa::ProcessingFailedError if _coordinator.processing_failed?
|
75
|
-
raise Dipa::TimeoutError if _coordinator.timed_out?
|
76
|
-
|
77
|
-
raise Dipa::UnknownProcessingStateError
|
78
34
|
end
|
79
35
|
|
80
|
-
def
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
result
|
36
|
+
def _result_and_cleanup
|
37
|
+
_wait_for_it_with_timeout
|
38
|
+
ensure
|
39
|
+
_after_result_action
|
86
40
|
end
|
87
41
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
def _fetch_result
|
93
|
-
_want_result? ? _coordinator.result : _coordinator.source
|
42
|
+
def _after_result_action
|
43
|
+
Dipa::CoordinatorServices::MaybeCleanupService.call(
|
44
|
+
coordinator: _coordinator
|
45
|
+
)
|
94
46
|
end
|
95
47
|
|
96
48
|
def _start_process
|
@@ -98,44 +50,6 @@ module Dipa
|
|
98
50
|
coordinator: _coordinator
|
99
51
|
)
|
100
52
|
end
|
101
|
-
|
102
|
-
def _prepare_coordinator(processor_class:, processor_method:)
|
103
|
-
@_coordinator = Dipa::Coordinator.create!(
|
104
|
-
agent_queue: _agent_queue,
|
105
|
-
coordinator_queue: _coordinator_queue,
|
106
|
-
keep_data: _keep_data?,
|
107
|
-
processor_class_name: processor_class,
|
108
|
-
processor_method_name: processor_method,
|
109
|
-
size: _source.to_a.size,
|
110
|
-
want_result: _want_result?
|
111
|
-
)
|
112
|
-
|
113
|
-
_coordinator.dump_to_file(data: _source.to_a, attacher: :source_dump)
|
114
|
-
end
|
115
|
-
|
116
|
-
def _agent_queue
|
117
|
-
_option(option: :agent_queue)
|
118
|
-
end
|
119
|
-
|
120
|
-
def _coordinator_queue
|
121
|
-
_option(option: :coordinator_queue)
|
122
|
-
end
|
123
|
-
|
124
|
-
def _async?
|
125
|
-
_option(option: :async)
|
126
|
-
end
|
127
|
-
|
128
|
-
def _keep_data?
|
129
|
-
_option(option: :keep_data)
|
130
|
-
end
|
131
|
-
|
132
|
-
def _want_result?
|
133
|
-
_option(option: :want_result)
|
134
|
-
end
|
135
|
-
|
136
|
-
def _option(option:)
|
137
|
-
OVERRIDE_OPTIONS[option] || _raw_opts[option] || DEFAULT_OPTIONS[option]
|
138
|
-
end
|
139
53
|
end
|
140
54
|
end
|
141
55
|
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Processor
|
5
|
+
module Concerns
|
6
|
+
module Coordinator
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
include Dipa::Processor::Concerns::Options
|
10
|
+
include Dipa::Processor::Concerns::Source
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
included do
|
15
|
+
attr_reader :_coordinator
|
16
|
+
end
|
17
|
+
|
18
|
+
def _prepare_coordinator(processor_class:, processor_method:)
|
19
|
+
_validate_processor_arguments(processor_class: processor_class,
|
20
|
+
processor_method: processor_method)
|
21
|
+
|
22
|
+
_create_coordinator(
|
23
|
+
processor_class_name: processor_class,
|
24
|
+
processor_method_name: processor_method
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
def _validate_processor_arguments(processor_class:, processor_method:)
|
29
|
+
return if processor_class.constantize.respond_to?(processor_method)
|
30
|
+
|
31
|
+
raise Dipa::UnknownProcessorMethodError,
|
32
|
+
"Method .#{processor_method} does not exist on processor " \
|
33
|
+
"class #{processor_class}"
|
34
|
+
rescue NameError => e
|
35
|
+
raise Dipa::UnknownProcessorClassError, e.original_message
|
36
|
+
end
|
37
|
+
|
38
|
+
def _create_coordinator(processor_class_name:, processor_method_name:)
|
39
|
+
@_coordinator = Dipa::Coordinator.create!(
|
40
|
+
_coordinator_create_params.merge(
|
41
|
+
{
|
42
|
+
processor_class_name: processor_class_name,
|
43
|
+
processor_method_name: processor_method_name
|
44
|
+
}
|
45
|
+
)
|
46
|
+
)
|
47
|
+
|
48
|
+
_coordinator.dump_to_file(data: _source.to_a, attacher: :source_dump)
|
49
|
+
end
|
50
|
+
|
51
|
+
def _coordinator_create_params
|
52
|
+
{
|
53
|
+
agent_queue: _agent_queue,
|
54
|
+
coordinator_queue: _coordinator_queue,
|
55
|
+
size: _source.to_a.size,
|
56
|
+
coordinator_options_record_attributes:
|
57
|
+
_coordinator_options_record_create_params
|
58
|
+
}
|
59
|
+
end
|
60
|
+
|
61
|
+
def _coordinator_options_record_create_params
|
62
|
+
{
|
63
|
+
keep_data: _keep_data?,
|
64
|
+
want_result: _want_result?,
|
65
|
+
agent_timeout: _agent_timeout,
|
66
|
+
agent_processing_timeout: _agent_processing_timeout,
|
67
|
+
coordinator_timeout: _coordinator_timeout,
|
68
|
+
coordinator_processing_timeout: _coordinator_processing_timeout
|
69
|
+
}
|
70
|
+
end
|
71
|
+
|
72
|
+
def _coordinator_waiting_state?
|
73
|
+
_coordinator.reload
|
74
|
+
|
75
|
+
!_coordinator.finished_state?
|
76
|
+
end
|
77
|
+
|
78
|
+
def _result
|
79
|
+
_coordinator.reload
|
80
|
+
|
81
|
+
_coordinator.want_result? ? _coordinator.result : _coordinator.source
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Processor
|
5
|
+
module Concerns
|
6
|
+
module Options
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
include ActiveSupport::Configurable
|
10
|
+
|
11
|
+
DEFAULT_ASYNC = false
|
12
|
+
DEFAULT_KEEP_DATA = false
|
13
|
+
DEFAULT_WANT_RESULT = true
|
14
|
+
|
15
|
+
OPTIONS_MAPPING = {
|
16
|
+
# queue names
|
17
|
+
agent_queue: { accessor: :_agent_queue, default: Dipa.agent_queue },
|
18
|
+
coordinator_queue: { accessor: :_coordinator_queue, default: Dipa.coordinator_queue },
|
19
|
+
# timeouts
|
20
|
+
agent_timeout: { accessor: :_agent_timeout, default: Dipa.agent_timeout },
|
21
|
+
agent_processing_timeout: { accessor: :_agent_processing_timeout, default: Dipa.agent_processing_timeout },
|
22
|
+
coordinator_timeout: { accessor: :_coordinator_timeout, default: Dipa.coordinator_timeout },
|
23
|
+
coordinator_processing_timeout: { accessor: :_coordinator_processing_timeout,
|
24
|
+
default: Dipa.coordinator_processing_timeout },
|
25
|
+
# misc
|
26
|
+
keep_data: { accessor: :_keep_data, default: DEFAULT_KEEP_DATA },
|
27
|
+
want_result: { accessor: :_want_result, default: DEFAULT_WANT_RESULT }
|
28
|
+
}.freeze
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
included do
|
33
|
+
OPTIONS_MAPPING.each_value do |args|
|
34
|
+
# Since ActiveSupport < 7 does not implement `default:`` option
|
35
|
+
# for config_accessor, defaults are set explicitly with
|
36
|
+
# `#_write_unprovided_options`
|
37
|
+
config_accessor(args[:accessor])
|
38
|
+
end
|
39
|
+
|
40
|
+
alias_method :_keep_data?, :_keep_data
|
41
|
+
alias_method :_want_result?, :_want_result
|
42
|
+
end
|
43
|
+
|
44
|
+
def _validate_options(options:)
|
45
|
+
options.assert_valid_keys(*OPTIONS_MAPPING.keys)
|
46
|
+
end
|
47
|
+
|
48
|
+
def _write_options(options:)
|
49
|
+
_write_provided_options(options: options)
|
50
|
+
_write_unprovided_options(options: options)
|
51
|
+
end
|
52
|
+
|
53
|
+
def _write_provided_options(options:)
|
54
|
+
options.each { |option, value| _write_option(option, value) }
|
55
|
+
end
|
56
|
+
|
57
|
+
def _write_unprovided_options(options:)
|
58
|
+
unprovided_options = OPTIONS_MAPPING.except(*options.keys)
|
59
|
+
|
60
|
+
unprovided_options.each do |option, args|
|
61
|
+
_write_option(option, args[:default])
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def _write_option(option, value)
|
66
|
+
new_value = _value_with_override_check(option, value)
|
67
|
+
__send__("#{OPTIONS_MAPPING[option][:accessor]}=", new_value)
|
68
|
+
end
|
69
|
+
|
70
|
+
def _value_with_override_check(option, value)
|
71
|
+
if self.class::OVERRIDE_OPTIONS.key?(option)
|
72
|
+
self.class::OVERRIDE_OPTIONS[option]
|
73
|
+
else
|
74
|
+
value
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Processor
|
5
|
+
module Concerns
|
6
|
+
module State
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def _processing_state?
|
12
|
+
@_processing_state ? true : false
|
13
|
+
end
|
14
|
+
|
15
|
+
def _check_state
|
16
|
+
raise Dipa::AlreadyProcessingError if _processing_state?
|
17
|
+
|
18
|
+
@_processing_state = true
|
19
|
+
end
|
20
|
+
|
21
|
+
def _reset_state
|
22
|
+
@_processing_state = false
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Processor
|
5
|
+
module Concerns
|
6
|
+
module Wait
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
include Dipa::Processor::Concerns::Coordinator
|
10
|
+
include Dipa::Processor::Concerns::Options
|
11
|
+
|
12
|
+
SYNC_MODE_SLEEP_START_DURATION = 2
|
13
|
+
SYNC_MODE_MAX_SLEEP_DURATION = 120
|
14
|
+
SYNC_MODE_SLEEP_DURATION_INCREASE_FACTOR = 1.1
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def _wait_for_it_with_timeout
|
19
|
+
_init_timeout_and_sleep_variables
|
20
|
+
|
21
|
+
_wait_for_it
|
22
|
+
end
|
23
|
+
|
24
|
+
def _init_timeout_and_sleep_variables
|
25
|
+
@_start = Time.current
|
26
|
+
@_sleep_duration = SYNC_MODE_SLEEP_START_DURATION
|
27
|
+
end
|
28
|
+
|
29
|
+
def _wait_for_it
|
30
|
+
_sleep while _coordinator_waiting_state?
|
31
|
+
|
32
|
+
return _result if _coordinator.finished?
|
33
|
+
|
34
|
+
# must be an error then
|
35
|
+
_raise_error
|
36
|
+
end
|
37
|
+
|
38
|
+
def _sleep
|
39
|
+
sleep(@_sleep_duration)
|
40
|
+
|
41
|
+
_adjust_sleep_duration
|
42
|
+
end
|
43
|
+
|
44
|
+
def _adjust_sleep_duration
|
45
|
+
return if @_sleep_duration > SYNC_MODE_MAX_SLEEP_DURATION
|
46
|
+
|
47
|
+
@_sleep_duration *= SYNC_MODE_SLEEP_DURATION_INCREASE_FACTOR
|
48
|
+
end
|
49
|
+
|
50
|
+
def _raise_error
|
51
|
+
if _coordinator.failed_state?
|
52
|
+
raise(
|
53
|
+
"Dipa::Coordinator#{_coordinator.state.to_s.camelize}Error".constantize
|
54
|
+
)
|
55
|
+
end
|
56
|
+
|
57
|
+
raise Dipa::UnknownProcessingStateError
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/dipa/processor/map.rb
CHANGED
data/lib/dipa/version.rb
CHANGED
data/lib/dipa.rb
CHANGED
@@ -9,14 +9,22 @@ require 'dipa/errors'
|
|
9
9
|
module Dipa
|
10
10
|
extend ActiveSupport::Autoload
|
11
11
|
|
12
|
+
# DEFAULT_TIMEOUT = 0
|
12
13
|
DEFAULT_AGENT_TIMEOUT = 0
|
14
|
+
DEFAULT_AGENT_PROCESSING_TIMEOUT = 0
|
13
15
|
DEFAULT_COORDINATOR_TIMEOUT = 0
|
16
|
+
DEFAULT_COORDINATOR_PROCESSING_TIMEOUT = 0
|
17
|
+
|
18
|
+
DATETIME_PRECISION = 6
|
14
19
|
|
15
20
|
# rubocop:disable ThreadSafety/ClassAndModuleAttributes
|
16
21
|
mattr_accessor :agent_queue
|
17
22
|
mattr_accessor :agent_timeout, default: DEFAULT_AGENT_TIMEOUT
|
23
|
+
mattr_accessor :agent_processing_timeout, default: DEFAULT_AGENT_PROCESSING_TIMEOUT
|
18
24
|
mattr_accessor :coordinator_queue
|
19
25
|
mattr_accessor :coordinator_timeout, default: DEFAULT_COORDINATOR_TIMEOUT
|
26
|
+
mattr_accessor :coordinator_processing_timeout, default: DEFAULT_COORDINATOR_PROCESSING_TIMEOUT
|
27
|
+
mattr_accessor :datetime_precision, default: DATETIME_PRECISION
|
20
28
|
# rubocop:enable ThreadSafety/ClassAndModuleAttributes
|
21
29
|
|
22
30
|
def self.map(source, options: {})
|
@@ -30,6 +38,16 @@ module Dipa
|
|
30
38
|
module Processor
|
31
39
|
extend ActiveSupport::Autoload
|
32
40
|
|
41
|
+
module Concerns
|
42
|
+
extend ActiveSupport::Autoload
|
43
|
+
|
44
|
+
autoload :Coordinator
|
45
|
+
autoload :Options
|
46
|
+
autoload :Source
|
47
|
+
autoload :State
|
48
|
+
autoload :Wait
|
49
|
+
end
|
50
|
+
|
33
51
|
autoload :Base
|
34
52
|
autoload :Each
|
35
53
|
autoload :Map
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dipa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0.pre.
|
4
|
+
version: 0.1.0.pre.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Merten Falk
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 6.
|
19
|
+
version: 6.1.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 8.0.0
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">"
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 6.
|
29
|
+
version: 6.1.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 8.0.0
|
@@ -36,7 +36,7 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - ">"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: 6.
|
39
|
+
version: 6.1.0
|
40
40
|
- - "<"
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: 8.0.0
|
@@ -46,7 +46,7 @@ dependencies:
|
|
46
46
|
requirements:
|
47
47
|
- - ">"
|
48
48
|
- !ruby/object:Gem::Version
|
49
|
-
version: 6.
|
49
|
+
version: 6.1.0
|
50
50
|
- - "<"
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 8.0.0
|
@@ -56,7 +56,7 @@ dependencies:
|
|
56
56
|
requirements:
|
57
57
|
- - ">"
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
version: 6.
|
59
|
+
version: 6.1.0
|
60
60
|
- - "<"
|
61
61
|
- !ruby/object:Gem::Version
|
62
62
|
version: 8.0.0
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ">"
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: 6.
|
69
|
+
version: 6.1.0
|
70
70
|
- - "<"
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 8.0.0
|
@@ -76,7 +76,7 @@ dependencies:
|
|
76
76
|
requirements:
|
77
77
|
- - ">"
|
78
78
|
- !ruby/object:Gem::Version
|
79
|
-
version: 6.
|
79
|
+
version: 6.1.0
|
80
80
|
- - "<"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: 8.0.0
|
@@ -86,7 +86,7 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - ">"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 6.
|
89
|
+
version: 6.1.0
|
90
90
|
- - "<"
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: 8.0.0
|
@@ -104,21 +104,33 @@ dependencies:
|
|
104
104
|
- - "~>"
|
105
105
|
- !ruby/object:Gem::Version
|
106
106
|
version: '13.0'
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: validates_timeliness
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ">="
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
type: :runtime
|
115
|
+
prerelease: false
|
116
|
+
version_requirements: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - ">="
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
107
121
|
description: |
|
108
|
-
This gem provides an API for parallel processing like the
|
109
|
-
|
110
|
-
|
111
|
-
|
122
|
+
This gem provides an API for parallel processing like the parallel gem but
|
123
|
+
distributed and scalable over different machines. All this with minimum
|
124
|
+
configuration and minimum dependencies to specific technologies and using
|
125
|
+
the rails ecosystem.
|
112
126
|
|
113
|
-
Dipa provides a rails engine which depends on
|
114
|
-
[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
|
115
|
-
[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
|
127
|
+
Dipa provides a rails engine which depends on ActiveJob and ActiveStorage.
|
116
128
|
You can use whatever backend you like for any of this components and
|
117
129
|
configure them for your specific usecase.
|
118
130
|
|
119
131
|
The purpose of this gem is to distribute load heavy and long running
|
120
132
|
processing of large datasets over multiple processes or machines using
|
121
|
-
|
133
|
+
ActiveJob.
|
122
134
|
email:
|
123
135
|
- empunkt@mailbox.org
|
124
136
|
executables: []
|
@@ -130,36 +142,53 @@ files:
|
|
130
142
|
- Rakefile
|
131
143
|
- app/jobs/dipa/application_job.rb
|
132
144
|
- app/jobs/dipa/service_job.rb
|
133
|
-
- app/models/concerns/models/
|
134
|
-
- app/models/concerns/models/
|
135
|
-
- app/models/concerns/models/
|
145
|
+
- app/models/concerns/dipa/models/agent_state_attributes.rb
|
146
|
+
- app/models/concerns/dipa/models/coordinator_state_attributes.rb
|
147
|
+
- app/models/concerns/dipa/models/dumpable.rb
|
148
|
+
- app/models/concerns/dipa/models/loadable.rb
|
149
|
+
- app/models/concerns/dipa/models/setting_constants.rb
|
150
|
+
- app/models/concerns/dipa/models/state_attribute_handling.rb
|
136
151
|
- app/models/dipa/agent.rb
|
137
152
|
- app/models/dipa/application_record.rb
|
138
153
|
- app/models/dipa/coordinator.rb
|
139
|
-
- app/models/
|
154
|
+
- app/models/dipa/coordinator_options_record.rb
|
155
|
+
- app/services/concerns/dipa/services/process_failure_handling.rb
|
156
|
+
- app/services/dipa/agent_services/base_service.rb
|
140
157
|
- app/services/dipa/agent_services/coordinator_state_service.rb
|
141
158
|
- app/services/dipa/agent_services/post_processing_service.rb
|
142
159
|
- app/services/dipa/agent_services/processing_service.rb
|
143
160
|
- app/services/dipa/agent_services/start_processing_service.rb
|
144
161
|
- app/services/dipa/application_service.rb
|
162
|
+
- app/services/dipa/coordinator_services/base_service.rb
|
145
163
|
- app/services/dipa/coordinator_services/create_agents_service.rb
|
164
|
+
- app/services/dipa/coordinator_services/destroy_service.rb
|
165
|
+
- app/services/dipa/coordinator_services/maybe_cleanup_service.rb
|
146
166
|
- app/services/dipa/coordinator_services/start_processing_service.rb
|
147
167
|
- app/validators/dipa/date_validator.rb
|
148
168
|
- db/migrate/20220102132652_create_dipa_coordinators.rb
|
149
169
|
- db/migrate/20220106183616_create_dipa_agents.rb
|
170
|
+
- db/migrate/20230624053724_create_dipa_coordinator_options_record.rb
|
150
171
|
- lib/dipa.rb
|
151
172
|
- lib/dipa/engine.rb
|
152
173
|
- lib/dipa/errors.rb
|
153
174
|
- lib/dipa/processor/base.rb
|
175
|
+
- lib/dipa/processor/concerns/coordinator.rb
|
176
|
+
- lib/dipa/processor/concerns/options.rb
|
177
|
+
- lib/dipa/processor/concerns/source.rb
|
178
|
+
- lib/dipa/processor/concerns/state.rb
|
179
|
+
- lib/dipa/processor/concerns/wait.rb
|
154
180
|
- lib/dipa/processor/each.rb
|
155
181
|
- lib/dipa/processor/map.rb
|
156
182
|
- lib/dipa/version.rb
|
157
|
-
- lib/tasks/auto_annotate_models.rake
|
158
183
|
- lib/tasks/dipa_tasks.rake
|
159
184
|
homepage: https://codeberg.org/empunkt/dipa
|
160
185
|
licenses:
|
161
186
|
- MIT
|
162
187
|
metadata:
|
188
|
+
allowed_push_host: https://rubygems.org
|
189
|
+
homepage_uri: https://codeberg.org/empunkt/dipa
|
190
|
+
source_code_uri: https://codeberg.org/empunkt/dipa
|
191
|
+
changelog_uri: https://codeberg.org/empunkt/dipa/src/branch/main/CHANGELOG.md
|
163
192
|
rubygems_mfa_required: 'true'
|
164
193
|
post_install_message:
|
165
194
|
rdoc_options: []
|
@@ -169,14 +198,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
169
198
|
requirements:
|
170
199
|
- - ">="
|
171
200
|
- !ruby/object:Gem::Version
|
172
|
-
version:
|
201
|
+
version: 3.0.0
|
173
202
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
174
203
|
requirements:
|
175
204
|
- - ">"
|
176
205
|
- !ruby/object:Gem::Version
|
177
206
|
version: 1.3.1
|
178
207
|
requirements: []
|
179
|
-
rubygems_version: 3.
|
208
|
+
rubygems_version: 3.4.14
|
180
209
|
signing_key:
|
181
210
|
specification_version: 4
|
182
211
|
summary: Rails engine that provides an API to execute code in parallel and distributed
|