dipa 0.1.0.pre.1 → 0.1.0.pre.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +77 -49
- data/Rakefile +50 -0
- data/app/models/concerns/dipa/models/agent_state_attributes.rb +24 -0
- data/app/models/concerns/dipa/models/coordinator_state_attributes.rb +24 -0
- data/app/models/concerns/{models/dipa → dipa/models}/dumpable.rb +3 -3
- data/app/models/concerns/{models/dipa → dipa/models}/loadable.rb +4 -4
- data/app/models/concerns/dipa/models/setting_constants.rb +23 -0
- data/app/models/concerns/dipa/models/state_attribute_handling.rb +77 -0
- data/app/models/dipa/agent.rb +14 -13
- data/app/models/dipa/coordinator.rb +32 -19
- data/app/models/dipa/coordinator_options_record.rb +52 -0
- data/app/services/concerns/dipa/services/process_failure_handling.rb +80 -0
- data/app/services/dipa/agent_services/base_service.rb +21 -0
- data/app/services/dipa/agent_services/coordinator_state_service.rb +58 -4
- data/app/services/dipa/agent_services/post_processing_service.rb +5 -3
- data/app/services/dipa/agent_services/processing_service.rb +84 -3
- data/app/services/dipa/agent_services/start_processing_service.rb +6 -4
- data/app/services/dipa/coordinator_services/base_service.rb +21 -0
- data/app/services/dipa/coordinator_services/create_agents_service.rb +7 -5
- data/app/services/dipa/coordinator_services/destroy_service.rb +13 -0
- data/app/services/dipa/coordinator_services/maybe_cleanup_service.rb +20 -0
- data/app/services/dipa/coordinator_services/start_processing_service.rb +6 -4
- data/db/migrate/20220102132652_create_dipa_coordinators.rb +3 -5
- data/db/migrate/20220106183616_create_dipa_agents.rb +3 -3
- data/db/migrate/20230624053724_create_dipa_coordinator_options_record.rb +21 -0
- data/lib/dipa/engine.rb +15 -8
- data/lib/dipa/errors.rb +20 -3
- data/lib/dipa/processor/base.rb +20 -106
- data/lib/dipa/processor/concerns/coordinator.rb +86 -0
- data/lib/dipa/processor/concerns/options.rb +80 -0
- data/lib/dipa/processor/concerns/source.rb +17 -0
- data/lib/dipa/processor/concerns/state.rb +27 -0
- data/lib/dipa/processor/concerns/wait.rb +62 -0
- data/lib/dipa/processor/map.rb +1 -2
- data/lib/dipa/version.rb +3 -1
- data/lib/dipa.rb +18 -0
- metadata +54 -25
- data/app/models/concerns/models/dipa/state_attribute_handling.rb +0 -38
- data/app/models/modules/models/dipa/status_constants.rb +0 -24
- data/lib/tasks/auto_annotate_models.rake +0 -62
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e27263056de7d6f8727bcd7bfad4f15bbf9e2e07672d07df687845d462032a7e
|
4
|
+
data.tar.gz: 3cc2fe847141a3e40a690ae7afe3769314f390c5eecd6e1f2c4f028eab0149b4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4a42b7fa9b406bedcfc17135e8c15219c6f7aa928ea1b02309a7bb55579689177d3d83e3951ca0080599522a4d0f6f7cef48a50e330009f216b183c7b49d744d
|
7
|
+
data.tar.gz: decbe534571620493f1133de0430987b421c8302435f69aeb29f5480e74bc3b8f5cd834832c124e4928ff2126321b097b0131d76b32e1d251549996607dc725e
|
data/README.md
CHANGED
@@ -2,27 +2,21 @@
|
|
2
2
|
|
3
3
|
# Dipa
|
4
4
|
|
5
|
-
This gem provides an API for parallel processing like the [parallel
|
6
|
-
|
7
|
-
|
8
|
-
to specific technologies and using the rails ecosystem.
|
5
|
+
This gem provides an API for parallel processing like the [parallel gem](https://github.com/grosser/parallel) but
|
6
|
+
distributed and scalable over different machines. All this with minimum configuration and minimum dependencies to
|
7
|
+
specific technologies and using the rails ecosystem.
|
9
8
|
|
10
|
-
Dipa provides a rails engine which depends on
|
11
|
-
[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
|
9
|
+
Dipa provides a rails engine which depends on [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
|
12
10
|
[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
|
13
|
-
You can use whatever backend you like for any of this components and configure
|
14
|
-
them for your specific usecase.
|
11
|
+
You can use whatever backend you like for any of this components and configure them for your specific usecase.
|
15
12
|
|
16
|
-
The purpose of this gem is to distribute load heavy and long running processing
|
17
|
-
|
18
|
-
[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
|
13
|
+
The purpose of this gem is to distribute load heavy and long running processing of large datasets over multiple
|
14
|
+
processes or machines using [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
|
19
15
|
|
20
16
|
## Installation
|
21
17
|
|
22
|
-
Before you install Dipa make sure
|
23
|
-
[
|
24
|
-
[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are
|
25
|
-
installed and configured properly.
|
18
|
+
Before you install Dipa make sure [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
|
19
|
+
[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are installed and configured properly.
|
26
20
|
|
27
21
|
Add this line to your application's Gemfile:
|
28
22
|
|
@@ -42,21 +36,28 @@ $ gem install dipa
|
|
42
36
|
|
43
37
|
Install Dipa migrations
|
44
38
|
```bash
|
45
|
-
bundle exec rake
|
39
|
+
bundle exec rake dipa:install:migrations
|
46
40
|
bundle exec rake db:migrate
|
47
41
|
```
|
48
42
|
|
49
43
|
## Configuration
|
50
44
|
|
51
|
-
Dipa can be configured in the application config.
|
45
|
+
Dipa can be configured in the application config. These configuration options set the default for this installation.
|
52
46
|
|
53
47
|
```ruby
|
54
48
|
config.dipa.agent_queue = :default_queue_for_dipa_agent_jobs
|
55
49
|
config.dipa.coordinator_queue = :default_queue_for_coordinator_queue_jobs
|
50
|
+
config.dipa.agent_timeout = 900
|
51
|
+
config.dipa.agent_processing_timeout = 600
|
52
|
+
config.dipa.coordinator_timeout = 0
|
53
|
+
config.dipa.coordinator_processing_timeout = 18000
|
56
54
|
```
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
- `config.dipa.agent_queue` defaults to `config.active_job.default_queue_name`
|
56
|
+
- `config.dipa.coordinator_queue` defaults to `config.active_job.default_queue_name`
|
57
|
+
- `config.dipa.agent_timeout` defaults to 0 (no timeout).
|
58
|
+
- `config.dipa.agent_processing_timeout` defaults to 0 (no timeout).
|
59
|
+
- `config.dipa.coordinator_timeout` defaults to 0 (no timeout).
|
60
|
+
- `config.dipa.coordinator_processing_timeout` defaults to 0 (no timeout).
|
60
61
|
|
61
62
|
## Usage
|
62
63
|
|
@@ -67,8 +68,8 @@ Dipa.map(1..100).with('Integer', :sqrt)
|
|
67
68
|
|
68
69
|
More realistic examples:
|
69
70
|
```ruby
|
70
|
-
Dipa.map(large_dataset, options).with('ProcessorClassName', :processor_class_method)
|
71
|
-
Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_method)
|
71
|
+
Dipa.map(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
|
72
|
+
Dipa.each(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
|
72
73
|
```
|
73
74
|
|
74
75
|
`Dipa.map` returns an `Array` of the processed items. The result is in the same order as the input (`large_dataset`).
|
@@ -79,22 +80,19 @@ Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_me
|
|
79
80
|
|
80
81
|
`options` is a hash. Following keys are allowed:
|
81
82
|
|
82
|
-
-
|
83
|
-
-
|
84
|
-
-
|
85
|
-
|
86
|
-
|
87
|
-
-
|
88
|
-
|
89
|
-
|
90
|
-
happen, set this to `true`.
|
83
|
+
- `agent_queue:` \[Symbol] Defaults to `config.dipa.agent_queue`.
|
84
|
+
- `coordinator_queue:` \[Symbol] Defaults to `config.dipa.coordinator_queue`.
|
85
|
+
- `agent_timeout:` \[Integer] Defaults to `config.dipa.agent_timeout`.
|
86
|
+
- `agent_processing_timeout:` \[Integer] Defaults to `config.dipa.agent_processing_timeout`.
|
87
|
+
- `coordinator_timeout:` \[Integer] Defaults to `config.dipa.coordinator_timeout`.
|
88
|
+
- `coordinator_processing_timeout:` \[Integer] Defaults to `config.dipa.coordinator_processing_timeout`.
|
89
|
+
- `keep_data:` \[true|false] Defaults to `false`. Useful for debugging. After processing all `Dipa::*` records and the
|
90
|
+
associated ActiveStorage data will be removed. If you don't want that to happen, set this to `true`.
|
91
91
|
|
92
|
-
`ProcessorClassName` must be a `Class` or a `String`. Defines the class which
|
93
|
-
provides the processor method.
|
92
|
+
`ProcessorClassName` must be a `Class` or a `String`. Defines the class which provides the processor method.
|
94
93
|
|
95
|
-
`:processor_class_method` must be a `Symbol` or a `String`. Defines the method
|
96
|
-
|
97
|
-
method. MUST except just one element as argument.
|
94
|
+
`:processor_class_method` must be a `Symbol` or a `String`. Defines the method which is used to process each single
|
95
|
+
element of `large_dataset`. MUST be a class method. MUST except just one element as argument.
|
98
96
|
|
99
97
|
## TODO
|
100
98
|
|
@@ -102,25 +100,55 @@ method. MUST except just one element as argument.
|
|
102
100
|
|
103
101
|
## Development
|
104
102
|
|
105
|
-
|
106
|
-
|
107
|
-
|
103
|
+
### With nix
|
104
|
+
|
105
|
+
- Having nix installed. See [https://nixos.org/download.html]() for detailed instructions for your OS.
|
106
|
+
- Enable flakes.
|
107
|
+
```shell
|
108
|
+
mkdir -p ~/.config/nix
|
109
|
+
echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
|
110
|
+
```
|
111
|
+
- **Optional but recommended:** Install direnv-nix as described
|
112
|
+
[here](https://github.com/nix-community/nix-direnv#installation)
|
113
|
+
- Clone this repository
|
114
|
+
- `cd` into the repository's directory
|
115
|
+
- Enter shell:
|
116
|
+
- Without direnv: Execute `nix develop`. You need to enter and leave the shell explicitly every time.
|
117
|
+
- With direnv:
|
118
|
+
```shell
|
119
|
+
echo -e "use flake . --impure" > .envrc
|
120
|
+
direnv allow
|
121
|
+
```
|
122
|
+
- Shell will start every time you `cd` into the projects directory and stop as soon as you leave it.
|
123
|
+
|
124
|
+
The shell sets up the environment for working with this repository and installs all required tools for this project.
|
125
|
+
|
126
|
+
Changes in the `flake.nix` fix will trigger a rebuild of your devenv environment as soon as you hit the shell
|
127
|
+
(return key/(re-)enter shell). Specifically, it rebuilds parts that needs rebuild only. You can also enforce a
|
128
|
+
rebuild by executing `direnv reload`.
|
129
|
+
|
130
|
+
Starting the shell the first time might take some minutes.
|
131
|
+
|
132
|
+
- Run `bundle install`.
|
133
|
+
- Start services in another terminal window with `devenv up` (as of 15.08.2023 it's mysql). The first
|
134
|
+
run will also setup the database.
|
135
|
+
- Run `bundle exec rake db:migrate`.
|
136
|
+
|
137
|
+
### Without nix
|
138
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests.
|
139
|
+
You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
108
140
|
|
109
141
|
## Contributing
|
110
142
|
|
111
|
-
Bug reports and pull requests are welcome on Codeberg at
|
112
|
-
|
113
|
-
|
114
|
-
the [code of
|
115
|
-
conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
|
143
|
+
Bug reports and pull requests are welcome on Codeberg at [https://codeberg.org/empunkt/dipa](). This project is intended
|
144
|
+
to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the
|
145
|
+
[code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
|
116
146
|
|
117
147
|
## License
|
118
148
|
|
119
|
-
The gem is available as open source under the terms of the
|
120
|
-
[MIT License](https://opensource.org/licenses/MIT).
|
149
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
121
150
|
|
122
151
|
## Code of Conduct
|
123
152
|
|
124
|
-
Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms
|
125
|
-
|
126
|
-
[code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
|
153
|
+
Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms and mailing lists is expected to follow
|
154
|
+
the [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
|
data/Rakefile
CHANGED
@@ -7,4 +7,54 @@ load 'rails/tasks/engine.rake'
|
|
7
7
|
|
8
8
|
load 'rails/tasks/statistics.rake'
|
9
9
|
|
10
|
+
require 'annotate'
|
11
|
+
Annotate.set_defaults(
|
12
|
+
'active_admin' => 'false',
|
13
|
+
'additional_file_patterns' => [],
|
14
|
+
'routes' => 'false',
|
15
|
+
'models' => 'true',
|
16
|
+
'position_in_routes' => 'before',
|
17
|
+
'position_in_class' => 'before',
|
18
|
+
'position_in_test' => 'before',
|
19
|
+
'position_in_fixture' => 'before',
|
20
|
+
'position_in_factory' => 'before',
|
21
|
+
'position_in_serializer' => 'before',
|
22
|
+
'show_foreign_keys' => 'true',
|
23
|
+
'show_complete_foreign_keys' => 'false',
|
24
|
+
'show_indexes' => 'true',
|
25
|
+
'simple_indexes' => 'false',
|
26
|
+
'model_dir' => 'app/models',
|
27
|
+
'root_dir' => '',
|
28
|
+
'include_version' => 'false',
|
29
|
+
'require' => '',
|
30
|
+
'exclude_tests' => 'true',
|
31
|
+
'exclude_fixtures' => 'true',
|
32
|
+
'exclude_factories' => 'false',
|
33
|
+
'exclude_serializers' => 'false',
|
34
|
+
'exclude_scaffolds' => 'true',
|
35
|
+
'exclude_controllers' => 'true',
|
36
|
+
'exclude_helpers' => 'true',
|
37
|
+
'exclude_sti_subclasses' => 'false',
|
38
|
+
'ignore_model_sub_dir' => 'false',
|
39
|
+
'ignore_columns' => nil,
|
40
|
+
'ignore_routes' => nil,
|
41
|
+
'ignore_unknown_models' => 'false',
|
42
|
+
'hide_limit_column_types' => 'integer,bigint,boolean',
|
43
|
+
'hide_default_column_types' => 'json,jsonb,hstore',
|
44
|
+
'skip_on_db_migrate' => 'false',
|
45
|
+
'format_bare' => 'true',
|
46
|
+
'format_rdoc' => 'false',
|
47
|
+
'format_yard' => 'false',
|
48
|
+
'format_markdown' => 'false',
|
49
|
+
'sort' => 'true',
|
50
|
+
'force' => 'false',
|
51
|
+
'frozen' => 'false',
|
52
|
+
'classified_sort' => 'true',
|
53
|
+
'trace' => 'false',
|
54
|
+
'wrapper_open' => nil,
|
55
|
+
'wrapper_close' => nil,
|
56
|
+
'with_comment' => 'true'
|
57
|
+
)
|
58
|
+
Annotate.load_tasks
|
59
|
+
|
10
60
|
require 'bundler/gem_tasks'
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
|
+
module AgentStateAttributes
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
include Dipa::Models::SettingConstants
|
9
|
+
|
10
|
+
AGENT_EXTRA_FINISHED_STATES = {
|
11
|
+
processing_failed: 'processing_failed',
|
12
|
+
aborted_through_coordinator: 'aborted_through_coordinator',
|
13
|
+
aborted_through_coordinator_timed_out: 'aborted_through_coordinator_timed_out',
|
14
|
+
aborted_through_coordinator_processing_timed_out: 'aborted_through_coordinator_processing_timed_out'
|
15
|
+
}.freeze
|
16
|
+
AGENT_EXTRA_CONTINUING_STATES = {}.freeze
|
17
|
+
|
18
|
+
included do
|
19
|
+
set_constant(konst: :EXTRA_FINISHED_STATES, value: AGENT_EXTRA_FINISHED_STATES)
|
20
|
+
set_constant(konst: :EXTRA_CONTINUING_STATES, value: AGENT_EXTRA_CONTINUING_STATES)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
|
+
module CoordinatorStateAttributes
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
include Dipa::Models::SettingConstants
|
9
|
+
|
10
|
+
COORDINATOR_EXTRA_FINISHED_STATES = {
|
11
|
+
aborted_through_agent: 'aborted_through_agent',
|
12
|
+
aborted_through_agent_timed_out: 'aborted_through_agent_timed_out',
|
13
|
+
aborted_through_agent_processing_timed_out: 'aborted_through_agent_processing_timed_out',
|
14
|
+
aborted_through_agent_processing_failed: 'aborted_through_agent_processing_failed'
|
15
|
+
}.freeze
|
16
|
+
COORDINATOR_EXTRA_CONTINUING_STATES = {}.freeze
|
17
|
+
|
18
|
+
included do
|
19
|
+
set_constant(konst: :EXTRA_FINISHED_STATES, value: COORDINATOR_EXTRA_FINISHED_STATES)
|
20
|
+
set_constant(konst: :EXTRA_CONTINUING_STATES, value: COORDINATOR_EXTRA_CONTINUING_STATES)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
4
|
-
module
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
5
|
module Dumpable
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
@@ -9,7 +9,7 @@ module Models
|
|
9
9
|
io = StringIO.new(Marshal.dump(data), 'rb')
|
10
10
|
filename = "#{attacher}.dat"
|
11
11
|
|
12
|
-
|
12
|
+
__send__(attacher).attach(io: io, filename: filename)
|
13
13
|
end
|
14
14
|
end
|
15
15
|
end
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
4
|
-
module
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
5
|
module Loadable
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
def load_from_file(attacher:)
|
9
|
-
return unless
|
9
|
+
return unless __send__(attacher).attached?
|
10
10
|
|
11
11
|
Marshal.load( # rubocop:disable Security/MarshalLoad
|
12
|
-
|
12
|
+
__send__(attacher).download
|
13
13
|
)
|
14
14
|
end
|
15
15
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
|
+
module SettingConstants
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
class_methods do
|
9
|
+
def set_constant(konst:, value:)
|
10
|
+
return if const_defined?(konst)
|
11
|
+
|
12
|
+
const_set(konst, value)
|
13
|
+
end
|
14
|
+
|
15
|
+
def set_from_merged_constants(konst:, first_from:, second_from:)
|
16
|
+
merged = const_get(first_from).merge(const_get(second_from))
|
17
|
+
|
18
|
+
set_constant(konst: konst, value: merged)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dipa
|
4
|
+
module Models
|
5
|
+
module StateAttributeHandling
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
include Dipa::Models::SettingConstants
|
9
|
+
|
10
|
+
SHARED_FINISHED_STATES = {
|
11
|
+
aborted: 'aborted',
|
12
|
+
finished: 'finished',
|
13
|
+
processing_failed: 'processing_failed',
|
14
|
+
processing_timed_out: 'processing_timed_out',
|
15
|
+
timed_out: 'timed_out'
|
16
|
+
}.freeze
|
17
|
+
SHARED_CONTINUING_STATES = {
|
18
|
+
initialized: 'initialized',
|
19
|
+
processing: 'processing'
|
20
|
+
}.freeze
|
21
|
+
|
22
|
+
included do |base|
|
23
|
+
set_constant(konst: :SHARED_FINISHED_STATES, value: SHARED_FINISHED_STATES)
|
24
|
+
set_constant(konst: :SHARED_CONTINUING_STATES, value: SHARED_CONTINUING_STATES)
|
25
|
+
|
26
|
+
set_from_merged_constants(konst: :COMBINED_FINISHED_STATES,
|
27
|
+
first_from: :SHARED_FINISHED_STATES, second_from: :EXTRA_FINISHED_STATES)
|
28
|
+
set_from_merged_constants(konst: :COMBINED_CONTINUING_STATES,
|
29
|
+
first_from: :SHARED_CONTINUING_STATES, second_from: :EXTRA_CONTINUING_STATES)
|
30
|
+
set_from_merged_constants(konst: :COMBINED_STATES,
|
31
|
+
first_from: :COMBINED_FINISHED_STATES, second_from: :COMBINED_CONTINUING_STATES)
|
32
|
+
|
33
|
+
# :nocov: Only one branch can be covered here
|
34
|
+
if Rails.version >= '7'
|
35
|
+
enum :state, base::COMBINED_STATES, default: base::COMBINED_STATES[:initialized]
|
36
|
+
validates :state, comparison: { equal_to: base::COMBINED_STATES[:initialized] }, on: :create
|
37
|
+
else
|
38
|
+
enum state: base::COMBINED_STATES, _default: base::COMBINED_STATES[:initialized]
|
39
|
+
validates :state, inclusion: { in: [base::COMBINED_STATES[:initialized]] }, on: :create
|
40
|
+
end
|
41
|
+
# :nocov:
|
42
|
+
|
43
|
+
validates :state, presence: true, inclusion: { in: base::COMBINED_STATES.values }
|
44
|
+
|
45
|
+
before_validation :_set_started_at, if: :_processing_started?
|
46
|
+
before_validation :_set_finished_at, if: :_processing_finished?
|
47
|
+
end
|
48
|
+
|
49
|
+
def finished_state?
|
50
|
+
self.class::COMBINED_FINISHED_STATES.value?(state)
|
51
|
+
end
|
52
|
+
|
53
|
+
def failed_state?
|
54
|
+
self.class::COMBINED_FINISHED_STATES.except(:finished).value?(state)
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def _set_started_at
|
60
|
+
self.started_at = Time.current
|
61
|
+
end
|
62
|
+
|
63
|
+
def _set_finished_at
|
64
|
+
self.finished_at = Time.current
|
65
|
+
end
|
66
|
+
|
67
|
+
def _processing_started?
|
68
|
+
state_changed?(from: self.class::COMBINED_STATES[:initialized], to: self.class::COMBINED_STATES[:processing])
|
69
|
+
end
|
70
|
+
|
71
|
+
def _processing_finished?
|
72
|
+
state_changed?(from: self.class::COMBINED_STATES[:processing]) &&
|
73
|
+
self.class::COMBINED_FINISHED_STATES.value?(state_change[1])
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/app/models/dipa/agent.rb
CHANGED
@@ -24,12 +24,13 @@
|
|
24
24
|
|
25
25
|
module Dipa
|
26
26
|
class Agent < ApplicationRecord
|
27
|
-
include Models::
|
28
|
-
include Models::
|
29
|
-
include Models::
|
27
|
+
include Dipa::Models::Dumpable
|
28
|
+
include Dipa::Models::Loadable
|
29
|
+
include Dipa::Models::AgentStateAttributes
|
30
|
+
include Dipa::Models::StateAttributeHandling
|
30
31
|
|
31
32
|
# validation and default for `state` attribute is included by
|
32
|
-
# Models::
|
33
|
+
# Dipa::Models::StateAttributeHandling
|
33
34
|
|
34
35
|
validates :index, numericality: { only_integer: true }
|
35
36
|
|
@@ -43,7 +44,7 @@ module Dipa
|
|
43
44
|
has_one_attached :result_dump
|
44
45
|
|
45
46
|
def result
|
46
|
-
return unless
|
47
|
+
return unless finished?
|
47
48
|
|
48
49
|
load_from_file(attacher: :result_dump)
|
49
50
|
end
|
@@ -52,16 +53,16 @@ module Dipa
|
|
52
53
|
load_from_file(attacher: :source_dump)
|
53
54
|
end
|
54
55
|
|
55
|
-
def
|
56
|
-
|
57
|
-
.public_send(
|
58
|
-
coordinator.processor_method_name,
|
59
|
-
source
|
60
|
-
)
|
56
|
+
def timeout_reached?
|
57
|
+
return false if coordinator.agent_timeout.zero?
|
61
58
|
|
62
|
-
|
59
|
+
(created_at + coordinator.agent_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
|
60
|
+
end
|
61
|
+
|
62
|
+
def processing_timeout_reached?
|
63
|
+
return false if coordinator.agent_processing_timeout.zero?
|
63
64
|
|
64
|
-
|
65
|
+
(started_at + coordinator.agent_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
|
65
66
|
end
|
66
67
|
end
|
67
68
|
end
|
@@ -8,50 +8,51 @@
|
|
8
8
|
# agent_queue :string(255) not null
|
9
9
|
# coordinator_queue :string(255) not null
|
10
10
|
# finished_at :datetime
|
11
|
-
# keep_data :boolean default(FALSE), not null
|
12
11
|
# processor_class_name :string(255) not null
|
13
12
|
# processor_method_name :string(255) not null
|
14
13
|
# size :integer not null
|
15
14
|
# started_at :datetime
|
16
15
|
# state :string(255) not null
|
17
|
-
# want_result :boolean default(TRUE), not null
|
18
16
|
# created_at :datetime not null
|
19
17
|
# updated_at :datetime not null
|
20
18
|
#
|
21
19
|
|
22
20
|
module Dipa
|
23
21
|
class Coordinator < ApplicationRecord
|
24
|
-
include Models::
|
25
|
-
include Models::
|
26
|
-
include Models::
|
22
|
+
include Dipa::Models::Dumpable
|
23
|
+
include Dipa::Models::Loadable
|
24
|
+
include Dipa::Models::CoordinatorStateAttributes
|
25
|
+
include Dipa::Models::StateAttributeHandling
|
27
26
|
|
28
27
|
# validation and default for `state` attribute is included by
|
29
|
-
# Models::
|
30
|
-
|
31
|
-
attribute :keep_data, :boolean, default: false
|
32
|
-
attribute :want_result, :boolean, default: true
|
28
|
+
# Dipa::Models::StateAttributeHandling
|
33
29
|
|
34
30
|
validates :agent_queue, presence: true
|
35
31
|
validates :coordinator_queue, presence: true
|
36
32
|
|
37
33
|
validates :size, numericality: { only_integer: true }
|
38
|
-
|
39
34
|
validates :started_at, 'dipa/date' => true, allow_nil: true
|
40
35
|
validates :finished_at, 'dipa/date' => true, allow_nil: true
|
41
|
-
|
42
36
|
validates :processor_class_name, presence: true
|
43
37
|
validates :processor_method_name, presence: true
|
44
38
|
|
45
|
-
validates :keep_data, inclusion: [true, false]
|
46
|
-
validates :want_result, inclusion: [true, false]
|
47
|
-
|
48
39
|
has_many :agents, dependent: :destroy, inverse_of: :coordinator,
|
49
40
|
foreign_key: :dipa_coordinator_id
|
50
41
|
|
42
|
+
has_one :coordinator_options_record, dependent: :destroy,
|
43
|
+
inverse_of: :coordinator,
|
44
|
+
foreign_key: :dipa_coordinator_id
|
45
|
+
accepts_nested_attributes_for :coordinator_options_record
|
46
|
+
|
51
47
|
has_one_attached :source_dump
|
52
48
|
|
49
|
+
delegate :keep_data?, :want_result?,
|
50
|
+
:agent_timeout, :agent_processing_timeout,
|
51
|
+
:coordinator_timeout, :coordinator_processing_timeout,
|
52
|
+
to: :coordinator_options_record
|
53
|
+
|
53
54
|
def result
|
54
|
-
return unless
|
55
|
+
return unless finished?
|
55
56
|
|
56
57
|
_result_from_agents
|
57
58
|
end
|
@@ -60,8 +61,20 @@ module Dipa
|
|
60
61
|
load_from_file(attacher: :source_dump)
|
61
62
|
end
|
62
63
|
|
63
|
-
def
|
64
|
-
_all_agents_created? &&
|
64
|
+
def all_agents_created_and_finished?
|
65
|
+
_all_agents_created? && _all_agents_finished?
|
66
|
+
end
|
67
|
+
|
68
|
+
def timeout_reached?
|
69
|
+
return false if coordinator_timeout.zero?
|
70
|
+
|
71
|
+
(created_at + coordinator_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
|
72
|
+
end
|
73
|
+
|
74
|
+
def processing_timeout_reached?
|
75
|
+
return false if coordinator_processing_timeout.zero?
|
76
|
+
|
77
|
+
(started_at + coordinator_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
|
65
78
|
end
|
66
79
|
|
67
80
|
private
|
@@ -70,8 +83,8 @@ module Dipa
|
|
70
83
|
agents.with_attached_result_dump.order(:index).map(&:result)
|
71
84
|
end
|
72
85
|
|
73
|
-
def
|
74
|
-
agents.all?(&:
|
86
|
+
def _all_agents_finished?
|
87
|
+
agents.all?(&:finished_state?)
|
75
88
|
end
|
76
89
|
|
77
90
|
def _all_agents_created?
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# == Schema Information
|
4
|
+
#
|
5
|
+
# Table name: dipa_coordinator_options_records
|
6
|
+
#
|
7
|
+
# id :bigint not null, primary key
|
8
|
+
# agent_processing_timeout :integer not null
|
9
|
+
# agent_timeout :integer not null
|
10
|
+
# coordinator_processing_timeout :integer not null
|
11
|
+
# coordinator_timeout :integer not null
|
12
|
+
# keep_data :boolean default(FALSE), not null
|
13
|
+
# want_result :boolean default(TRUE), not null
|
14
|
+
# created_at :datetime not null
|
15
|
+
# updated_at :datetime not null
|
16
|
+
# dipa_coordinator_id :bigint not null
|
17
|
+
#
|
18
|
+
# Indexes
|
19
|
+
#
|
20
|
+
# index_dipa_coordinator_options_records_on_dipa_coordinator_id (dipa_coordinator_id)
|
21
|
+
#
|
22
|
+
# Foreign Keys
|
23
|
+
#
|
24
|
+
# fk_rails_... (dipa_coordinator_id => dipa_coordinators.id)
|
25
|
+
#
|
26
|
+
|
27
|
+
module Dipa
|
28
|
+
class CoordinatorOptionsRecord < ApplicationRecord
|
29
|
+
attribute :keep_data, :boolean, default: false
|
30
|
+
attribute :want_result, :boolean, default: true
|
31
|
+
|
32
|
+
attribute :agent_timeout, :integer
|
33
|
+
attribute :agent_processing_timeout, :integer
|
34
|
+
attribute :coordinator_timeout, :integer
|
35
|
+
attribute :coordinator_processing_timeout, :integer
|
36
|
+
|
37
|
+
validates :keep_data, inclusion: [true, false]
|
38
|
+
validates :want_result, inclusion: [true, false]
|
39
|
+
|
40
|
+
validates :agent_timeout,
|
41
|
+
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
|
42
|
+
validates :agent_processing_timeout,
|
43
|
+
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
|
44
|
+
validates :coordinator_timeout,
|
45
|
+
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
|
46
|
+
validates :coordinator_processing_timeout,
|
47
|
+
numericality: { only_integer: true, greater_than_or_equal_to: 0 }
|
48
|
+
|
49
|
+
belongs_to :coordinator, inverse_of: :coordinator_options_record,
|
50
|
+
foreign_key: :dipa_coordinator_id
|
51
|
+
end
|
52
|
+
end
|