dipa 0.1.0.pre.1 → 0.1.0.pre.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +77 -49
  3. data/Rakefile +50 -0
  4. data/app/models/concerns/dipa/models/agent_state_attributes.rb +24 -0
  5. data/app/models/concerns/dipa/models/coordinator_state_attributes.rb +24 -0
  6. data/app/models/concerns/{models/dipa → dipa/models}/dumpable.rb +3 -3
  7. data/app/models/concerns/{models/dipa → dipa/models}/loadable.rb +4 -4
  8. data/app/models/concerns/dipa/models/setting_constants.rb +23 -0
  9. data/app/models/concerns/dipa/models/state_attribute_handling.rb +77 -0
  10. data/app/models/dipa/agent.rb +14 -13
  11. data/app/models/dipa/coordinator.rb +32 -19
  12. data/app/models/dipa/coordinator_options_record.rb +52 -0
  13. data/app/services/concerns/dipa/services/process_failure_handling.rb +80 -0
  14. data/app/services/dipa/agent_services/base_service.rb +21 -0
  15. data/app/services/dipa/agent_services/coordinator_state_service.rb +58 -4
  16. data/app/services/dipa/agent_services/post_processing_service.rb +5 -3
  17. data/app/services/dipa/agent_services/processing_service.rb +84 -3
  18. data/app/services/dipa/agent_services/start_processing_service.rb +6 -4
  19. data/app/services/dipa/coordinator_services/base_service.rb +21 -0
  20. data/app/services/dipa/coordinator_services/create_agents_service.rb +7 -5
  21. data/app/services/dipa/coordinator_services/destroy_service.rb +13 -0
  22. data/app/services/dipa/coordinator_services/maybe_cleanup_service.rb +20 -0
  23. data/app/services/dipa/coordinator_services/start_processing_service.rb +6 -4
  24. data/db/migrate/20220102132652_create_dipa_coordinators.rb +3 -5
  25. data/db/migrate/20220106183616_create_dipa_agents.rb +3 -3
  26. data/db/migrate/20230624053724_create_dipa_coordinator_options_record.rb +21 -0
  27. data/lib/dipa/engine.rb +15 -8
  28. data/lib/dipa/errors.rb +20 -3
  29. data/lib/dipa/processor/base.rb +20 -106
  30. data/lib/dipa/processor/concerns/coordinator.rb +86 -0
  31. data/lib/dipa/processor/concerns/options.rb +80 -0
  32. data/lib/dipa/processor/concerns/source.rb +17 -0
  33. data/lib/dipa/processor/concerns/state.rb +27 -0
  34. data/lib/dipa/processor/concerns/wait.rb +62 -0
  35. data/lib/dipa/processor/map.rb +1 -2
  36. data/lib/dipa/version.rb +3 -1
  37. data/lib/dipa.rb +18 -0
  38. metadata +54 -25
  39. data/app/models/concerns/models/dipa/state_attribute_handling.rb +0 -38
  40. data/app/models/modules/models/dipa/status_constants.rb +0 -24
  41. data/lib/tasks/auto_annotate_models.rake +0 -62
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6403004fa28c6c0ae9f9cc40f9da8f5b812ed5ea4feb94bd00210785dd4bb950
4
- data.tar.gz: 5d485e3096f6b835520844bf6564354eb07640159f4fc7579566df78c8c7c999
3
+ metadata.gz: e27263056de7d6f8727bcd7bfad4f15bbf9e2e07672d07df687845d462032a7e
4
+ data.tar.gz: 3cc2fe847141a3e40a690ae7afe3769314f390c5eecd6e1f2c4f028eab0149b4
5
5
  SHA512:
6
- metadata.gz: 4903b379f5772adf30f2791c74e6355a3a753bb8808a3d8bf655790e7d601e925806d110ec3561993979b3f3aee9a19c5fade0a7d2b17afc7d856e3b5709fecf
7
- data.tar.gz: 4022abee161ffd77df8a8a11f9712285689287b369d994334c0bc82ca0f6615f087d1f9aeafa184536b09e399a74921e26d15e670274e98ae836d7aff033c1fd
6
+ metadata.gz: 4a42b7fa9b406bedcfc17135e8c15219c6f7aa928ea1b02309a7bb55579689177d3d83e3951ca0080599522a4d0f6f7cef48a50e330009f216b183c7b49d744d
7
+ data.tar.gz: decbe534571620493f1133de0430987b421c8302435f69aeb29f5480e74bc3b8f5cd834832c124e4928ff2126321b097b0131d76b32e1d251549996607dc725e
data/README.md CHANGED
@@ -2,27 +2,21 @@
2
2
 
3
3
  # Dipa
4
4
 
5
- This gem provides an API for parallel processing like the [parallel
6
- gem](https://github.com/grosser/parallel) but distributed and scalable over
7
- different machines. All this with minimum configuration and minimum dependencies
8
- to specific technologies and using the rails ecosystem.
5
+ This gem provides an API for parallel processing like the [parallel gem](https://github.com/grosser/parallel) but
6
+ distributed and scalable over different machines. All this with minimum configuration and minimum dependencies to
7
+ specific technologies and using the rails ecosystem.
9
8
 
10
- Dipa provides a rails engine which depends on
11
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
9
+ Dipa provides a rails engine which depends on [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
12
10
  [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
13
- You can use whatever backend you like for any of this components and configure
14
- them for your specific usecase.
11
+ You can use whatever backend you like for any of this components and configure them for your specific usecase.
15
12
 
16
- The purpose of this gem is to distribute load heavy and long running processing
17
- of large datasets over multiple processes or machines using
18
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
13
+ The purpose of this gem is to distribute load heavy and long running processing of large datasets over multiple
14
+ processes or machines using [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
19
15
 
20
16
  ## Installation
21
17
 
22
- Before you install Dipa make sure
23
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
24
- [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are
25
- installed and configured properly.
18
+ Before you install Dipa make sure [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
19
+ [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are installed and configured properly.
26
20
 
27
21
  Add this line to your application's Gemfile:
28
22
 
@@ -42,21 +36,28 @@ $ gem install dipa
42
36
 
43
37
  Install Dipa migrations
44
38
  ```bash
45
- bundle exec rake app:dipa:install:migrations
39
+ bundle exec rake dipa:install:migrations
46
40
  bundle exec rake db:migrate
47
41
  ```
48
42
 
49
43
  ## Configuration
50
44
 
51
- Dipa can be configured in the application config.
45
+ Dipa can be configured in the application config. These configuration options set the default for this installation.
52
46
 
53
47
  ```ruby
54
48
  config.dipa.agent_queue = :default_queue_for_dipa_agent_jobs
55
49
  config.dipa.coordinator_queue = :default_queue_for_coordinator_queue_jobs
50
+ config.dipa.agent_timeout = 900
51
+ config.dipa.agent_processing_timeout = 600
52
+ config.dipa.coordinator_timeout = 0
53
+ config.dipa.coordinator_processing_timeout = 18000
56
54
  ```
57
-
58
- If not configured `config.active_job.default_queue_name` or `:default` will be
59
- used.
55
+ - `config.dipa.agent_queue` defaults to `config.active_job.default_queue_name`
56
+ - `config.dipa.coordinator_queue` defaults to `config.active_job.default_queue_name`
57
+ - `config.dipa.agent_timeout` defaults to 0 (no timeout).
58
+ - `config.dipa.agent_processing_timeout` defaults to 0 (no timeout).
59
+ - `config.dipa.coordinator_timeout` defaults to 0 (no timeout).
60
+ - `config.dipa.coordinator_processing_timeout` defaults to 0 (no timeout).
60
61
 
61
62
  ## Usage
62
63
 
@@ -67,8 +68,8 @@ Dipa.map(1..100).with('Integer', :sqrt)
67
68
 
68
69
  More realistic examples:
69
70
  ```ruby
70
- Dipa.map(large_dataset, options).with('ProcessorClassName', :processor_class_method)
71
- Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_method)
71
+ Dipa.map(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
72
+ Dipa.each(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
72
73
  ```
73
74
 
74
75
  `Dipa.map` returns an `Array` of the processed items. The result is in the same order as the input (`large_dataset`).
@@ -79,22 +80,19 @@ Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_me
79
80
 
80
81
  `options` is a hash. Following keys are allowed:
81
82
 
82
- - `agent_queue:` [Symbol] Defaults to `config.dipa.agent_queue`.
83
- - `coordinator_queue:` [Symbol] Defaults to `config.dipa.coordinator_queue`.
84
- - `async:` [true|false] Defaults to `false`. Usually no need to change, but
85
- could be useful for `Dipa.each` if you have a alternative way to monitor your
86
- jobs.
87
- - `keep_data:` [true|false] Defaults to `false`. Useful for debugging. After
88
- processing all `Dipa::Agent` and `Dipa::Coordinator` records and the
89
- associated ActiveStorage data will be removed. If you don't want that to
90
- happen, set this to `true`.
83
+ - `agent_queue:` \[Symbol] Defaults to `config.dipa.agent_queue`.
84
+ - `coordinator_queue:` \[Symbol] Defaults to `config.dipa.coordinator_queue`.
85
+ - `agent_timeout:` \[Integer] Defaults to `config.dipa.agent_timeout`.
86
+ - `agent_processing_timeout:` \[Integer] Defaults to `config.dipa.agent_processing_timeout`.
87
+ - `coordinator_timeout:` \[Integer] Defaults to `config.dipa.coordinator_timeout`.
88
+ - `coordinator_processing_timeout:` \[Integer] Defaults to `config.dipa.coordinator_processing_timeout`.
89
+ - `keep_data:` \[true|false] Defaults to `false`. Useful for debugging. After processing all `Dipa::*` records and the
90
+ associated ActiveStorage data will be removed. If you don't want that to happen, set this to `true`.
91
91
 
92
- `ProcessorClassName` must be a `Class` or a `String`. Defines the class which
93
- provides the processor method.
92
+ `ProcessorClassName` must be a `Class` or a `String`. Defines the class which provides the processor method.
94
93
 
95
- `:processor_class_method` must be a `Symbol` or a `String`. Defines the method
96
- which is used to process each single element of `large_dataset`. MUST be a class
97
- method. MUST except just one element as argument.
94
+ `:processor_class_method` must be a `Symbol` or a `String`. Defines the method which is used to process each single
95
+ element of `large_dataset`. MUST be a class method. MUST except just one element as argument.
98
96
 
99
97
  ## TODO
100
98
 
@@ -102,25 +100,55 @@ method. MUST except just one element as argument.
102
100
 
103
101
  ## Development
104
102
 
105
- After checking out the repo, run `bin/setup` to install dependencies. Then, run
106
- `bundle exec rspec` to run the tests. You can also run `bin/console` for an
107
- interactive prompt that will allow you to experiment.
103
+ ### With nix
104
+
105
+ - Having nix installed. See [https://nixos.org/download.html]() for detailed instructions for your OS.
106
+ - Enable flakes.
107
+ ```shell
108
+ mkdir -p ~/.config/nix
109
+ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
110
+ ```
111
+ - **Optional but recommended:** Install direnv-nix as described
112
+ [here](https://github.com/nix-community/nix-direnv#installation)
113
+ - Clone this repository
114
+ - `cd` into the repository's directory
115
+ - Enter shell:
116
+ - Without direnv: Execute `nix develop`. You need to enter and leave the shell explicitly every time.
117
+ - With direnv:
118
+ ```shell
119
+ echo -e "use flake . --impure" > .envrc
120
+ direnv allow
121
+ ```
122
+ - Shell will start every time you `cd` into the projects directory and stop as soon as you leave it.
123
+
124
+ The shell sets up the environment for working with this repository and installs all required tools for this project.
125
+
126
+ Changes in the `flake.nix` fix will trigger a rebuild of your devenv environment as soon as you hit the shell
127
+ (return key/(re-)enter shell). Specifically, it rebuilds parts that needs rebuild only. You can also enforce a
128
+ rebuild by executing `direnv reload`.
129
+
130
+ Starting the shell the first time might take some minutes.
131
+
132
+ - Run `bundle install`.
133
+ - Start services in another terminal window with `devenv up` (as of 15.08.2023 it's mysql). The first
134
+ run will also setup the database.
135
+ - Run `bundle exec rake db:migrate`.
136
+
137
+ ### Without nix
138
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests.
139
+ You can also run `bin/console` for an interactive prompt that will allow you to experiment.
108
140
 
109
141
  ## Contributing
110
142
 
111
- Bug reports and pull requests are welcome on Codeberg at
112
- https://codeberg.org/empunkt/dipa. This project is intended to be a safe,
113
- welcoming space for collaboration, and contributors are expected to adhere to
114
- the [code of
115
- conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
143
+ Bug reports and pull requests are welcome on Codeberg at [https://codeberg.org/empunkt/dipa](). This project is intended
144
+ to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the
145
+ [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
116
146
 
117
147
  ## License
118
148
 
119
- The gem is available as open source under the terms of the
120
- [MIT License](https://opensource.org/licenses/MIT).
149
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
121
150
 
122
151
  ## Code of Conduct
123
152
 
124
- Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms
125
- and mailing lists is expected to follow the
126
- [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
153
+ Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms and mailing lists is expected to follow
154
+ the [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
data/Rakefile CHANGED
@@ -7,4 +7,54 @@ load 'rails/tasks/engine.rake'
7
7
 
8
8
  load 'rails/tasks/statistics.rake'
9
9
 
10
+ require 'annotate'
11
+ Annotate.set_defaults(
12
+ 'active_admin' => 'false',
13
+ 'additional_file_patterns' => [],
14
+ 'routes' => 'false',
15
+ 'models' => 'true',
16
+ 'position_in_routes' => 'before',
17
+ 'position_in_class' => 'before',
18
+ 'position_in_test' => 'before',
19
+ 'position_in_fixture' => 'before',
20
+ 'position_in_factory' => 'before',
21
+ 'position_in_serializer' => 'before',
22
+ 'show_foreign_keys' => 'true',
23
+ 'show_complete_foreign_keys' => 'false',
24
+ 'show_indexes' => 'true',
25
+ 'simple_indexes' => 'false',
26
+ 'model_dir' => 'app/models',
27
+ 'root_dir' => '',
28
+ 'include_version' => 'false',
29
+ 'require' => '',
30
+ 'exclude_tests' => 'true',
31
+ 'exclude_fixtures' => 'true',
32
+ 'exclude_factories' => 'false',
33
+ 'exclude_serializers' => 'false',
34
+ 'exclude_scaffolds' => 'true',
35
+ 'exclude_controllers' => 'true',
36
+ 'exclude_helpers' => 'true',
37
+ 'exclude_sti_subclasses' => 'false',
38
+ 'ignore_model_sub_dir' => 'false',
39
+ 'ignore_columns' => nil,
40
+ 'ignore_routes' => nil,
41
+ 'ignore_unknown_models' => 'false',
42
+ 'hide_limit_column_types' => 'integer,bigint,boolean',
43
+ 'hide_default_column_types' => 'json,jsonb,hstore',
44
+ 'skip_on_db_migrate' => 'false',
45
+ 'format_bare' => 'true',
46
+ 'format_rdoc' => 'false',
47
+ 'format_yard' => 'false',
48
+ 'format_markdown' => 'false',
49
+ 'sort' => 'true',
50
+ 'force' => 'false',
51
+ 'frozen' => 'false',
52
+ 'classified_sort' => 'true',
53
+ 'trace' => 'false',
54
+ 'wrapper_open' => nil,
55
+ 'wrapper_close' => nil,
56
+ 'with_comment' => 'true'
57
+ )
58
+ Annotate.load_tasks
59
+
10
60
  require 'bundler/gem_tasks'
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module AgentStateAttributes
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ AGENT_EXTRA_FINISHED_STATES = {
11
+ processing_failed: 'processing_failed',
12
+ aborted_through_coordinator: 'aborted_through_coordinator',
13
+ aborted_through_coordinator_timed_out: 'aborted_through_coordinator_timed_out',
14
+ aborted_through_coordinator_processing_timed_out: 'aborted_through_coordinator_processing_timed_out'
15
+ }.freeze
16
+ AGENT_EXTRA_CONTINUING_STATES = {}.freeze
17
+
18
+ included do
19
+ set_constant(konst: :EXTRA_FINISHED_STATES, value: AGENT_EXTRA_FINISHED_STATES)
20
+ set_constant(konst: :EXTRA_CONTINUING_STATES, value: AGENT_EXTRA_CONTINUING_STATES)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module CoordinatorStateAttributes
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ COORDINATOR_EXTRA_FINISHED_STATES = {
11
+ aborted_through_agent: 'aborted_through_agent',
12
+ aborted_through_agent_timed_out: 'aborted_through_agent_timed_out',
13
+ aborted_through_agent_processing_timed_out: 'aborted_through_agent_processing_timed_out',
14
+ aborted_through_agent_processing_failed: 'aborted_through_agent_processing_failed'
15
+ }.freeze
16
+ COORDINATOR_EXTRA_CONTINUING_STATES = {}.freeze
17
+
18
+ included do
19
+ set_constant(konst: :EXTRA_FINISHED_STATES, value: COORDINATOR_EXTRA_FINISHED_STATES)
20
+ set_constant(konst: :EXTRA_CONTINUING_STATES, value: COORDINATOR_EXTRA_CONTINUING_STATES)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Models
4
- module Dipa
3
+ module Dipa
4
+ module Models
5
5
  module Dumpable
6
6
  extend ActiveSupport::Concern
7
7
 
@@ -9,7 +9,7 @@ module Models
9
9
  io = StringIO.new(Marshal.dump(data), 'rb')
10
10
  filename = "#{attacher}.dat"
11
11
 
12
- public_send(attacher).attach(io: io, filename: filename)
12
+ __send__(attacher).attach(io: io, filename: filename)
13
13
  end
14
14
  end
15
15
  end
@@ -1,15 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Models
4
- module Dipa
3
+ module Dipa
4
+ module Models
5
5
  module Loadable
6
6
  extend ActiveSupport::Concern
7
7
 
8
8
  def load_from_file(attacher:)
9
- return unless public_send(attacher).attached?
9
+ return unless __send__(attacher).attached?
10
10
 
11
11
  Marshal.load( # rubocop:disable Security/MarshalLoad
12
- public_send(attacher).download
12
+ __send__(attacher).download
13
13
  )
14
14
  end
15
15
  end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module SettingConstants
6
+ extend ActiveSupport::Concern
7
+
8
+ class_methods do
9
+ def set_constant(konst:, value:)
10
+ return if const_defined?(konst)
11
+
12
+ const_set(konst, value)
13
+ end
14
+
15
+ def set_from_merged_constants(konst:, first_from:, second_from:)
16
+ merged = const_get(first_from).merge(const_get(second_from))
17
+
18
+ set_constant(konst: konst, value: merged)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module StateAttributeHandling
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ SHARED_FINISHED_STATES = {
11
+ aborted: 'aborted',
12
+ finished: 'finished',
13
+ processing_failed: 'processing_failed',
14
+ processing_timed_out: 'processing_timed_out',
15
+ timed_out: 'timed_out'
16
+ }.freeze
17
+ SHARED_CONTINUING_STATES = {
18
+ initialized: 'initialized',
19
+ processing: 'processing'
20
+ }.freeze
21
+
22
+ included do |base|
23
+ set_constant(konst: :SHARED_FINISHED_STATES, value: SHARED_FINISHED_STATES)
24
+ set_constant(konst: :SHARED_CONTINUING_STATES, value: SHARED_CONTINUING_STATES)
25
+
26
+ set_from_merged_constants(konst: :COMBINED_FINISHED_STATES,
27
+ first_from: :SHARED_FINISHED_STATES, second_from: :EXTRA_FINISHED_STATES)
28
+ set_from_merged_constants(konst: :COMBINED_CONTINUING_STATES,
29
+ first_from: :SHARED_CONTINUING_STATES, second_from: :EXTRA_CONTINUING_STATES)
30
+ set_from_merged_constants(konst: :COMBINED_STATES,
31
+ first_from: :COMBINED_FINISHED_STATES, second_from: :COMBINED_CONTINUING_STATES)
32
+
33
+ # :nocov: Only one branch can be covered here
34
+ if Rails.version >= '7'
35
+ enum :state, base::COMBINED_STATES, default: base::COMBINED_STATES[:initialized]
36
+ validates :state, comparison: { equal_to: base::COMBINED_STATES[:initialized] }, on: :create
37
+ else
38
+ enum state: base::COMBINED_STATES, _default: base::COMBINED_STATES[:initialized]
39
+ validates :state, inclusion: { in: [base::COMBINED_STATES[:initialized]] }, on: :create
40
+ end
41
+ # :nocov:
42
+
43
+ validates :state, presence: true, inclusion: { in: base::COMBINED_STATES.values }
44
+
45
+ before_validation :_set_started_at, if: :_processing_started?
46
+ before_validation :_set_finished_at, if: :_processing_finished?
47
+ end
48
+
49
+ def finished_state?
50
+ self.class::COMBINED_FINISHED_STATES.value?(state)
51
+ end
52
+
53
+ def failed_state?
54
+ self.class::COMBINED_FINISHED_STATES.except(:finished).value?(state)
55
+ end
56
+
57
+ private
58
+
59
+ def _set_started_at
60
+ self.started_at = Time.current
61
+ end
62
+
63
+ def _set_finished_at
64
+ self.finished_at = Time.current
65
+ end
66
+
67
+ def _processing_started?
68
+ state_changed?(from: self.class::COMBINED_STATES[:initialized], to: self.class::COMBINED_STATES[:processing])
69
+ end
70
+
71
+ def _processing_finished?
72
+ state_changed?(from: self.class::COMBINED_STATES[:processing]) &&
73
+ self.class::COMBINED_FINISHED_STATES.value?(state_change[1])
74
+ end
75
+ end
76
+ end
77
+ end
@@ -24,12 +24,13 @@
24
24
 
25
25
  module Dipa
26
26
  class Agent < ApplicationRecord
27
- include Models::Dipa::Dumpable
28
- include Models::Dipa::Loadable
29
- include Models::Dipa::StateAttributeHandling
27
+ include Dipa::Models::Dumpable
28
+ include Dipa::Models::Loadable
29
+ include Dipa::Models::AgentStateAttributes
30
+ include Dipa::Models::StateAttributeHandling
30
31
 
31
32
  # validation and default for `state` attribute is included by
32
- # Models::Dipa::StateAttributeHandling
33
+ # Dipa::Models::StateAttributeHandling
33
34
 
34
35
  validates :index, numericality: { only_integer: true }
35
36
 
@@ -43,7 +44,7 @@ module Dipa
43
44
  has_one_attached :result_dump
44
45
 
45
46
  def result
46
- return unless processed?
47
+ return unless finished?
47
48
 
48
49
  load_from_file(attacher: :result_dump)
49
50
  end
@@ -52,16 +53,16 @@ module Dipa
52
53
  load_from_file(attacher: :source_dump)
53
54
  end
54
55
 
55
- def process!
56
- processor_result = coordinator.processor_class_name.constantize
57
- .public_send(
58
- coordinator.processor_method_name,
59
- source
60
- )
56
+ def timeout_reached?
57
+ return false if coordinator.agent_timeout.zero?
61
58
 
62
- dump_to_file(data: processor_result, attacher: :result_dump)
59
+ (created_at + coordinator.agent_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
60
+ end
61
+
62
+ def processing_timeout_reached?
63
+ return false if coordinator.agent_processing_timeout.zero?
63
64
 
64
- finished!
65
+ (started_at + coordinator.agent_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
65
66
  end
66
67
  end
67
68
  end
@@ -8,50 +8,51 @@
8
8
  # agent_queue :string(255) not null
9
9
  # coordinator_queue :string(255) not null
10
10
  # finished_at :datetime
11
- # keep_data :boolean default(FALSE), not null
12
11
  # processor_class_name :string(255) not null
13
12
  # processor_method_name :string(255) not null
14
13
  # size :integer not null
15
14
  # started_at :datetime
16
15
  # state :string(255) not null
17
- # want_result :boolean default(TRUE), not null
18
16
  # created_at :datetime not null
19
17
  # updated_at :datetime not null
20
18
  #
21
19
 
22
20
  module Dipa
23
21
  class Coordinator < ApplicationRecord
24
- include Models::Dipa::Dumpable
25
- include Models::Dipa::Loadable
26
- include Models::Dipa::StateAttributeHandling
22
+ include Dipa::Models::Dumpable
23
+ include Dipa::Models::Loadable
24
+ include Dipa::Models::CoordinatorStateAttributes
25
+ include Dipa::Models::StateAttributeHandling
27
26
 
28
27
  # validation and default for `state` attribute is included by
29
- # Models::Dipa::StateAttributeHandling
30
-
31
- attribute :keep_data, :boolean, default: false
32
- attribute :want_result, :boolean, default: true
28
+ # Dipa::Models::StateAttributeHandling
33
29
 
34
30
  validates :agent_queue, presence: true
35
31
  validates :coordinator_queue, presence: true
36
32
 
37
33
  validates :size, numericality: { only_integer: true }
38
-
39
34
  validates :started_at, 'dipa/date' => true, allow_nil: true
40
35
  validates :finished_at, 'dipa/date' => true, allow_nil: true
41
-
42
36
  validates :processor_class_name, presence: true
43
37
  validates :processor_method_name, presence: true
44
38
 
45
- validates :keep_data, inclusion: [true, false]
46
- validates :want_result, inclusion: [true, false]
47
-
48
39
  has_many :agents, dependent: :destroy, inverse_of: :coordinator,
49
40
  foreign_key: :dipa_coordinator_id
50
41
 
42
+ has_one :coordinator_options_record, dependent: :destroy,
43
+ inverse_of: :coordinator,
44
+ foreign_key: :dipa_coordinator_id
45
+ accepts_nested_attributes_for :coordinator_options_record
46
+
51
47
  has_one_attached :source_dump
52
48
 
49
+ delegate :keep_data?, :want_result?,
50
+ :agent_timeout, :agent_processing_timeout,
51
+ :coordinator_timeout, :coordinator_processing_timeout,
52
+ to: :coordinator_options_record
53
+
53
54
  def result
54
- return unless processed?
55
+ return unless finished?
55
56
 
56
57
  _result_from_agents
57
58
  end
@@ -60,8 +61,20 @@ module Dipa
60
61
  load_from_file(attacher: :source_dump)
61
62
  end
62
63
 
63
- def all_agents_created_and_processed?
64
- _all_agents_created? && _all_agents_processed?
64
+ def all_agents_created_and_finished?
65
+ _all_agents_created? && _all_agents_finished?
66
+ end
67
+
68
+ def timeout_reached?
69
+ return false if coordinator_timeout.zero?
70
+
71
+ (created_at + coordinator_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
72
+ end
73
+
74
+ def processing_timeout_reached?
75
+ return false if coordinator_processing_timeout.zero?
76
+
77
+ (started_at + coordinator_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
65
78
  end
66
79
 
67
80
  private
@@ -70,8 +83,8 @@ module Dipa
70
83
  agents.with_attached_result_dump.order(:index).map(&:result)
71
84
  end
72
85
 
73
- def _all_agents_processed?
74
- agents.all?(&:processed?)
86
+ def _all_agents_finished?
87
+ agents.all?(&:finished_state?)
75
88
  end
76
89
 
77
90
  def _all_agents_created?
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ # == Schema Information
4
+ #
5
+ # Table name: dipa_coordinator_options_records
6
+ #
7
+ # id :bigint not null, primary key
8
+ # agent_processing_timeout :integer not null
9
+ # agent_timeout :integer not null
10
+ # coordinator_processing_timeout :integer not null
11
+ # coordinator_timeout :integer not null
12
+ # keep_data :boolean default(FALSE), not null
13
+ # want_result :boolean default(TRUE), not null
14
+ # created_at :datetime not null
15
+ # updated_at :datetime not null
16
+ # dipa_coordinator_id :bigint not null
17
+ #
18
+ # Indexes
19
+ #
20
+ # index_dipa_coordinator_options_records_on_dipa_coordinator_id (dipa_coordinator_id)
21
+ #
22
+ # Foreign Keys
23
+ #
24
+ # fk_rails_... (dipa_coordinator_id => dipa_coordinators.id)
25
+ #
26
+
27
+ module Dipa
28
+ class CoordinatorOptionsRecord < ApplicationRecord
29
+ attribute :keep_data, :boolean, default: false
30
+ attribute :want_result, :boolean, default: true
31
+
32
+ attribute :agent_timeout, :integer
33
+ attribute :agent_processing_timeout, :integer
34
+ attribute :coordinator_timeout, :integer
35
+ attribute :coordinator_processing_timeout, :integer
36
+
37
+ validates :keep_data, inclusion: [true, false]
38
+ validates :want_result, inclusion: [true, false]
39
+
40
+ validates :agent_timeout,
41
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
42
+ validates :agent_processing_timeout,
43
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
44
+ validates :coordinator_timeout,
45
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
46
+ validates :coordinator_processing_timeout,
47
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
48
+
49
+ belongs_to :coordinator, inverse_of: :coordinator_options_record,
50
+ foreign_key: :dipa_coordinator_id
51
+ end
52
+ end