dipa 0.1.0.pre.1 → 0.1.0.pre.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +77 -49
  3. data/Rakefile +50 -0
  4. data/app/models/concerns/dipa/models/agent_state_attributes.rb +24 -0
  5. data/app/models/concerns/dipa/models/coordinator_state_attributes.rb +24 -0
  6. data/app/models/concerns/{models/dipa → dipa/models}/dumpable.rb +3 -3
  7. data/app/models/concerns/{models/dipa → dipa/models}/loadable.rb +4 -4
  8. data/app/models/concerns/dipa/models/setting_constants.rb +23 -0
  9. data/app/models/concerns/dipa/models/state_attribute_handling.rb +77 -0
  10. data/app/models/dipa/agent.rb +14 -13
  11. data/app/models/dipa/coordinator.rb +32 -19
  12. data/app/models/dipa/coordinator_options_record.rb +52 -0
  13. data/app/services/concerns/dipa/services/process_failure_handling.rb +80 -0
  14. data/app/services/dipa/agent_services/base_service.rb +21 -0
  15. data/app/services/dipa/agent_services/coordinator_state_service.rb +58 -4
  16. data/app/services/dipa/agent_services/post_processing_service.rb +5 -3
  17. data/app/services/dipa/agent_services/processing_service.rb +84 -3
  18. data/app/services/dipa/agent_services/start_processing_service.rb +6 -4
  19. data/app/services/dipa/coordinator_services/base_service.rb +21 -0
  20. data/app/services/dipa/coordinator_services/create_agents_service.rb +7 -5
  21. data/app/services/dipa/coordinator_services/destroy_service.rb +13 -0
  22. data/app/services/dipa/coordinator_services/maybe_cleanup_service.rb +20 -0
  23. data/app/services/dipa/coordinator_services/start_processing_service.rb +6 -4
  24. data/db/migrate/20220102132652_create_dipa_coordinators.rb +3 -5
  25. data/db/migrate/20220106183616_create_dipa_agents.rb +3 -3
  26. data/db/migrate/20230624053724_create_dipa_coordinator_options_record.rb +21 -0
  27. data/lib/dipa/engine.rb +15 -8
  28. data/lib/dipa/errors.rb +20 -3
  29. data/lib/dipa/processor/base.rb +20 -106
  30. data/lib/dipa/processor/concerns/coordinator.rb +86 -0
  31. data/lib/dipa/processor/concerns/options.rb +80 -0
  32. data/lib/dipa/processor/concerns/source.rb +17 -0
  33. data/lib/dipa/processor/concerns/state.rb +27 -0
  34. data/lib/dipa/processor/concerns/wait.rb +62 -0
  35. data/lib/dipa/processor/map.rb +1 -2
  36. data/lib/dipa/version.rb +3 -1
  37. data/lib/dipa.rb +18 -0
  38. metadata +54 -25
  39. data/app/models/concerns/models/dipa/state_attribute_handling.rb +0 -38
  40. data/app/models/modules/models/dipa/status_constants.rb +0 -24
  41. data/lib/tasks/auto_annotate_models.rake +0 -62
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6403004fa28c6c0ae9f9cc40f9da8f5b812ed5ea4feb94bd00210785dd4bb950
4
- data.tar.gz: 5d485e3096f6b835520844bf6564354eb07640159f4fc7579566df78c8c7c999
3
+ metadata.gz: e27263056de7d6f8727bcd7bfad4f15bbf9e2e07672d07df687845d462032a7e
4
+ data.tar.gz: 3cc2fe847141a3e40a690ae7afe3769314f390c5eecd6e1f2c4f028eab0149b4
5
5
  SHA512:
6
- metadata.gz: 4903b379f5772adf30f2791c74e6355a3a753bb8808a3d8bf655790e7d601e925806d110ec3561993979b3f3aee9a19c5fade0a7d2b17afc7d856e3b5709fecf
7
- data.tar.gz: 4022abee161ffd77df8a8a11f9712285689287b369d994334c0bc82ca0f6615f087d1f9aeafa184536b09e399a74921e26d15e670274e98ae836d7aff033c1fd
6
+ metadata.gz: 4a42b7fa9b406bedcfc17135e8c15219c6f7aa928ea1b02309a7bb55579689177d3d83e3951ca0080599522a4d0f6f7cef48a50e330009f216b183c7b49d744d
7
+ data.tar.gz: decbe534571620493f1133de0430987b421c8302435f69aeb29f5480e74bc3b8f5cd834832c124e4928ff2126321b097b0131d76b32e1d251549996607dc725e
data/README.md CHANGED
@@ -2,27 +2,21 @@
2
2
 
3
3
  # Dipa
4
4
 
5
- This gem provides an API for parallel processing like the [parallel
6
- gem](https://github.com/grosser/parallel) but distributed and scalable over
7
- different machines. All this with minimum configuration and minimum dependencies
8
- to specific technologies and using the rails ecosystem.
5
+ This gem provides an API for parallel processing like the [parallel gem](https://github.com/grosser/parallel) but
6
+ distributed and scalable over different machines. All this with minimum configuration and minimum dependencies to
7
+ specific technologies and using the rails ecosystem.
9
8
 
10
- Dipa provides a rails engine which depends on
11
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
9
+ Dipa provides a rails engine which depends on [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
12
10
  [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
13
- You can use whatever backend you like for any of this components and configure
14
- them for your specific usecase.
11
+ You can use whatever backend you like for any of this components and configure them for your specific usecase.
15
12
 
16
- The purpose of this gem is to distribute load heavy and long running processing
17
- of large datasets over multiple processes or machines using
18
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
13
+ The purpose of this gem is to distribute load heavy and long running processing of large datasets over multiple
14
+ processes or machines using [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
19
15
 
20
16
  ## Installation
21
17
 
22
- Before you install Dipa make sure
23
- [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
24
- [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are
25
- installed and configured properly.
18
+ Before you install Dipa make sure [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
19
+ [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are installed and configured properly.
26
20
 
27
21
  Add this line to your application's Gemfile:
28
22
 
@@ -42,21 +36,28 @@ $ gem install dipa
42
36
 
43
37
  Install Dipa migrations
44
38
  ```bash
45
- bundle exec rake app:dipa:install:migrations
39
+ bundle exec rake dipa:install:migrations
46
40
  bundle exec rake db:migrate
47
41
  ```
48
42
 
49
43
  ## Configuration
50
44
 
51
- Dipa can be configured in the application config.
45
+ Dipa can be configured in the application config. These configuration options set the default for this installation.
52
46
 
53
47
  ```ruby
54
48
  config.dipa.agent_queue = :default_queue_for_dipa_agent_jobs
55
49
  config.dipa.coordinator_queue = :default_queue_for_coordinator_queue_jobs
50
+ config.dipa.agent_timeout = 900
51
+ config.dipa.agent_processing_timeout = 600
52
+ config.dipa.coordinator_timeout = 0
53
+ config.dipa.coordinator_processing_timeout = 18000
56
54
  ```
57
-
58
- If not configured `config.active_job.default_queue_name` or `:default` will be
59
- used.
55
+ - `config.dipa.agent_queue` defaults to `config.active_job.default_queue_name`
56
+ - `config.dipa.coordinator_queue` defaults to `config.active_job.default_queue_name`
57
+ - `config.dipa.agent_timeout` defaults to 0 (no timeout).
58
+ - `config.dipa.agent_processing_timeout` defaults to 0 (no timeout).
59
+ - `config.dipa.coordinator_timeout` defaults to 0 (no timeout).
60
+ - `config.dipa.coordinator_processing_timeout` defaults to 0 (no timeout).
60
61
 
61
62
  ## Usage
62
63
 
@@ -67,8 +68,8 @@ Dipa.map(1..100).with('Integer', :sqrt)
67
68
 
68
69
  More realistic examples:
69
70
  ```ruby
70
- Dipa.map(large_dataset, options).with('ProcessorClassName', :processor_class_method)
71
- Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_method)
71
+ Dipa.map(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
72
+ Dipa.each(large_dataset, options: options).with('ProcessorClassName', :processor_class_method)
72
73
  ```
73
74
 
74
75
  `Dipa.map` returns an `Array` of the processed items. The result is in the same order as the input (`large_dataset`).
@@ -79,22 +80,19 @@ Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_me
79
80
 
80
81
  `options` is a hash. Following keys are allowed:
81
82
 
82
- - `agent_queue:` [Symbol] Defaults to `config.dipa.agent_queue`.
83
- - `coordinator_queue:` [Symbol] Defaults to `config.dipa.coordinator_queue`.
84
- - `async:` [true|false] Defaults to `false`. Usually no need to change, but
85
- could be useful for `Dipa.each` if you have a alternative way to monitor your
86
- jobs.
87
- - `keep_data:` [true|false] Defaults to `false`. Useful for debugging. After
88
- processing all `Dipa::Agent` and `Dipa::Coordinator` records and the
89
- associated ActiveStorage data will be removed. If you don't want that to
90
- happen, set this to `true`.
83
+ - `agent_queue:` \[Symbol] Defaults to `config.dipa.agent_queue`.
84
+ - `coordinator_queue:` \[Symbol] Defaults to `config.dipa.coordinator_queue`.
85
+ - `agent_timeout:` \[Integer] Defaults to `config.dipa.agent_timeout`.
86
+ - `agent_processing_timeout:` \[Integer] Defaults to `config.dipa.agent_processing_timeout`.
87
+ - `coordinator_timeout:` \[Integer] Defaults to `config.dipa.coordinator_timeout`.
88
+ - `coordinator_processing_timeout:` \[Integer] Defaults to `config.dipa.coordinator_processing_timeout`.
89
+ - `keep_data:` \[true|false] Defaults to `false`. Useful for debugging. After processing all `Dipa::*` records and the
90
+ associated ActiveStorage data will be removed. If you don't want that to happen, set this to `true`.
91
91
 
92
- `ProcessorClassName` must be a `Class` or a `String`. Defines the class which
93
- provides the processor method.
92
+ `ProcessorClassName` must be a `Class` or a `String`. Defines the class which provides the processor method.
94
93
 
95
- `:processor_class_method` must be a `Symbol` or a `String`. Defines the method
96
- which is used to process each single element of `large_dataset`. MUST be a class
97
- method. MUST except just one element as argument.
94
+ `:processor_class_method` must be a `Symbol` or a `String`. Defines the method which is used to process each single
95
+ element of `large_dataset`. MUST be a class method. MUST except just one element as argument.
98
96
 
99
97
  ## TODO
100
98
 
@@ -102,25 +100,55 @@ method. MUST except just one element as argument.
102
100
 
103
101
  ## Development
104
102
 
105
- After checking out the repo, run `bin/setup` to install dependencies. Then, run
106
- `bundle exec rspec` to run the tests. You can also run `bin/console` for an
107
- interactive prompt that will allow you to experiment.
103
+ ### With nix
104
+
105
+ - Having nix installed. See [https://nixos.org/download.html]() for detailed instructions for your OS.
106
+ - Enable flakes.
107
+ ```shell
108
+ mkdir -p ~/.config/nix
109
+ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
110
+ ```
111
+ - **Optional but recommended:** Install direnv-nix as described
112
+ [here](https://github.com/nix-community/nix-direnv#installation)
113
+ - Clone this repository
114
+ - `cd` into the repository's directory
115
+ - Enter shell:
116
+ - Without direnv: Execute `nix develop`. You need to enter and leave the shell explicitly every time.
117
+ - With direnv:
118
+ ```shell
119
+ echo -e "use flake . --impure" > .envrc
120
+ direnv allow
121
+ ```
122
+ - Shell will start every time you `cd` into the projects directory and stop as soon as you leave it.
123
+
124
+ The shell sets up the environment for working with this repository and installs all required tools for this project.
125
+
126
+ Changes in the `flake.nix` fix will trigger a rebuild of your devenv environment as soon as you hit the shell
127
+ (return key/(re-)enter shell). Specifically, it rebuilds parts that needs rebuild only. You can also enforce a
128
+ rebuild by executing `direnv reload`.
129
+
130
+ Starting the shell the first time might take some minutes.
131
+
132
+ - Run `bundle install`.
133
+ - Start services in another terminal window with `devenv up` (as of 15.08.2023 it's mysql). The first
134
+ run will also setup the database.
135
+ - Run `bundle exec rake db:migrate`.
136
+
137
+ ### Without nix
138
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests.
139
+ You can also run `bin/console` for an interactive prompt that will allow you to experiment.
108
140
 
109
141
  ## Contributing
110
142
 
111
- Bug reports and pull requests are welcome on Codeberg at
112
- https://codeberg.org/empunkt/dipa. This project is intended to be a safe,
113
- welcoming space for collaboration, and contributors are expected to adhere to
114
- the [code of
115
- conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
143
+ Bug reports and pull requests are welcome on Codeberg at [https://codeberg.org/empunkt/dipa](). This project is intended
144
+ to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the
145
+ [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
116
146
 
117
147
  ## License
118
148
 
119
- The gem is available as open source under the terms of the
120
- [MIT License](https://opensource.org/licenses/MIT).
149
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
121
150
 
122
151
  ## Code of Conduct
123
152
 
124
- Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms
125
- and mailing lists is expected to follow the
126
- [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
153
+ Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms and mailing lists is expected to follow
154
+ the [code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
data/Rakefile CHANGED
@@ -7,4 +7,54 @@ load 'rails/tasks/engine.rake'
7
7
 
8
8
  load 'rails/tasks/statistics.rake'
9
9
 
10
+ require 'annotate'
11
+ Annotate.set_defaults(
12
+ 'active_admin' => 'false',
13
+ 'additional_file_patterns' => [],
14
+ 'routes' => 'false',
15
+ 'models' => 'true',
16
+ 'position_in_routes' => 'before',
17
+ 'position_in_class' => 'before',
18
+ 'position_in_test' => 'before',
19
+ 'position_in_fixture' => 'before',
20
+ 'position_in_factory' => 'before',
21
+ 'position_in_serializer' => 'before',
22
+ 'show_foreign_keys' => 'true',
23
+ 'show_complete_foreign_keys' => 'false',
24
+ 'show_indexes' => 'true',
25
+ 'simple_indexes' => 'false',
26
+ 'model_dir' => 'app/models',
27
+ 'root_dir' => '',
28
+ 'include_version' => 'false',
29
+ 'require' => '',
30
+ 'exclude_tests' => 'true',
31
+ 'exclude_fixtures' => 'true',
32
+ 'exclude_factories' => 'false',
33
+ 'exclude_serializers' => 'false',
34
+ 'exclude_scaffolds' => 'true',
35
+ 'exclude_controllers' => 'true',
36
+ 'exclude_helpers' => 'true',
37
+ 'exclude_sti_subclasses' => 'false',
38
+ 'ignore_model_sub_dir' => 'false',
39
+ 'ignore_columns' => nil,
40
+ 'ignore_routes' => nil,
41
+ 'ignore_unknown_models' => 'false',
42
+ 'hide_limit_column_types' => 'integer,bigint,boolean',
43
+ 'hide_default_column_types' => 'json,jsonb,hstore',
44
+ 'skip_on_db_migrate' => 'false',
45
+ 'format_bare' => 'true',
46
+ 'format_rdoc' => 'false',
47
+ 'format_yard' => 'false',
48
+ 'format_markdown' => 'false',
49
+ 'sort' => 'true',
50
+ 'force' => 'false',
51
+ 'frozen' => 'false',
52
+ 'classified_sort' => 'true',
53
+ 'trace' => 'false',
54
+ 'wrapper_open' => nil,
55
+ 'wrapper_close' => nil,
56
+ 'with_comment' => 'true'
57
+ )
58
+ Annotate.load_tasks
59
+
10
60
  require 'bundler/gem_tasks'
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module AgentStateAttributes
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ AGENT_EXTRA_FINISHED_STATES = {
11
+ processing_failed: 'processing_failed',
12
+ aborted_through_coordinator: 'aborted_through_coordinator',
13
+ aborted_through_coordinator_timed_out: 'aborted_through_coordinator_timed_out',
14
+ aborted_through_coordinator_processing_timed_out: 'aborted_through_coordinator_processing_timed_out'
15
+ }.freeze
16
+ AGENT_EXTRA_CONTINUING_STATES = {}.freeze
17
+
18
+ included do
19
+ set_constant(konst: :EXTRA_FINISHED_STATES, value: AGENT_EXTRA_FINISHED_STATES)
20
+ set_constant(konst: :EXTRA_CONTINUING_STATES, value: AGENT_EXTRA_CONTINUING_STATES)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module CoordinatorStateAttributes
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ COORDINATOR_EXTRA_FINISHED_STATES = {
11
+ aborted_through_agent: 'aborted_through_agent',
12
+ aborted_through_agent_timed_out: 'aborted_through_agent_timed_out',
13
+ aborted_through_agent_processing_timed_out: 'aborted_through_agent_processing_timed_out',
14
+ aborted_through_agent_processing_failed: 'aborted_through_agent_processing_failed'
15
+ }.freeze
16
+ COORDINATOR_EXTRA_CONTINUING_STATES = {}.freeze
17
+
18
+ included do
19
+ set_constant(konst: :EXTRA_FINISHED_STATES, value: COORDINATOR_EXTRA_FINISHED_STATES)
20
+ set_constant(konst: :EXTRA_CONTINUING_STATES, value: COORDINATOR_EXTRA_CONTINUING_STATES)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Models
4
- module Dipa
3
+ module Dipa
4
+ module Models
5
5
  module Dumpable
6
6
  extend ActiveSupport::Concern
7
7
 
@@ -9,7 +9,7 @@ module Models
9
9
  io = StringIO.new(Marshal.dump(data), 'rb')
10
10
  filename = "#{attacher}.dat"
11
11
 
12
- public_send(attacher).attach(io: io, filename: filename)
12
+ __send__(attacher).attach(io: io, filename: filename)
13
13
  end
14
14
  end
15
15
  end
@@ -1,15 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Models
4
- module Dipa
3
+ module Dipa
4
+ module Models
5
5
  module Loadable
6
6
  extend ActiveSupport::Concern
7
7
 
8
8
  def load_from_file(attacher:)
9
- return unless public_send(attacher).attached?
9
+ return unless __send__(attacher).attached?
10
10
 
11
11
  Marshal.load( # rubocop:disable Security/MarshalLoad
12
- public_send(attacher).download
12
+ __send__(attacher).download
13
13
  )
14
14
  end
15
15
  end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module SettingConstants
6
+ extend ActiveSupport::Concern
7
+
8
+ class_methods do
9
+ def set_constant(konst:, value:)
10
+ return if const_defined?(konst)
11
+
12
+ const_set(konst, value)
13
+ end
14
+
15
+ def set_from_merged_constants(konst:, first_from:, second_from:)
16
+ merged = const_get(first_from).merge(const_get(second_from))
17
+
18
+ set_constant(konst: konst, value: merged)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dipa
4
+ module Models
5
+ module StateAttributeHandling
6
+ extend ActiveSupport::Concern
7
+
8
+ include Dipa::Models::SettingConstants
9
+
10
+ SHARED_FINISHED_STATES = {
11
+ aborted: 'aborted',
12
+ finished: 'finished',
13
+ processing_failed: 'processing_failed',
14
+ processing_timed_out: 'processing_timed_out',
15
+ timed_out: 'timed_out'
16
+ }.freeze
17
+ SHARED_CONTINUING_STATES = {
18
+ initialized: 'initialized',
19
+ processing: 'processing'
20
+ }.freeze
21
+
22
+ included do |base|
23
+ set_constant(konst: :SHARED_FINISHED_STATES, value: SHARED_FINISHED_STATES)
24
+ set_constant(konst: :SHARED_CONTINUING_STATES, value: SHARED_CONTINUING_STATES)
25
+
26
+ set_from_merged_constants(konst: :COMBINED_FINISHED_STATES,
27
+ first_from: :SHARED_FINISHED_STATES, second_from: :EXTRA_FINISHED_STATES)
28
+ set_from_merged_constants(konst: :COMBINED_CONTINUING_STATES,
29
+ first_from: :SHARED_CONTINUING_STATES, second_from: :EXTRA_CONTINUING_STATES)
30
+ set_from_merged_constants(konst: :COMBINED_STATES,
31
+ first_from: :COMBINED_FINISHED_STATES, second_from: :COMBINED_CONTINUING_STATES)
32
+
33
+ # :nocov: Only one branch can be covered here
34
+ if Rails.version >= '7'
35
+ enum :state, base::COMBINED_STATES, default: base::COMBINED_STATES[:initialized]
36
+ validates :state, comparison: { equal_to: base::COMBINED_STATES[:initialized] }, on: :create
37
+ else
38
+ enum state: base::COMBINED_STATES, _default: base::COMBINED_STATES[:initialized]
39
+ validates :state, inclusion: { in: [base::COMBINED_STATES[:initialized]] }, on: :create
40
+ end
41
+ # :nocov:
42
+
43
+ validates :state, presence: true, inclusion: { in: base::COMBINED_STATES.values }
44
+
45
+ before_validation :_set_started_at, if: :_processing_started?
46
+ before_validation :_set_finished_at, if: :_processing_finished?
47
+ end
48
+
49
+ def finished_state?
50
+ self.class::COMBINED_FINISHED_STATES.value?(state)
51
+ end
52
+
53
+ def failed_state?
54
+ self.class::COMBINED_FINISHED_STATES.except(:finished).value?(state)
55
+ end
56
+
57
+ private
58
+
59
+ def _set_started_at
60
+ self.started_at = Time.current
61
+ end
62
+
63
+ def _set_finished_at
64
+ self.finished_at = Time.current
65
+ end
66
+
67
+ def _processing_started?
68
+ state_changed?(from: self.class::COMBINED_STATES[:initialized], to: self.class::COMBINED_STATES[:processing])
69
+ end
70
+
71
+ def _processing_finished?
72
+ state_changed?(from: self.class::COMBINED_STATES[:processing]) &&
73
+ self.class::COMBINED_FINISHED_STATES.value?(state_change[1])
74
+ end
75
+ end
76
+ end
77
+ end
@@ -24,12 +24,13 @@
24
24
 
25
25
  module Dipa
26
26
  class Agent < ApplicationRecord
27
- include Models::Dipa::Dumpable
28
- include Models::Dipa::Loadable
29
- include Models::Dipa::StateAttributeHandling
27
+ include Dipa::Models::Dumpable
28
+ include Dipa::Models::Loadable
29
+ include Dipa::Models::AgentStateAttributes
30
+ include Dipa::Models::StateAttributeHandling
30
31
 
31
32
  # validation and default for `state` attribute is included by
32
- # Models::Dipa::StateAttributeHandling
33
+ # Dipa::Models::StateAttributeHandling
33
34
 
34
35
  validates :index, numericality: { only_integer: true }
35
36
 
@@ -43,7 +44,7 @@ module Dipa
43
44
  has_one_attached :result_dump
44
45
 
45
46
  def result
46
- return unless processed?
47
+ return unless finished?
47
48
 
48
49
  load_from_file(attacher: :result_dump)
49
50
  end
@@ -52,16 +53,16 @@ module Dipa
52
53
  load_from_file(attacher: :source_dump)
53
54
  end
54
55
 
55
- def process!
56
- processor_result = coordinator.processor_class_name.constantize
57
- .public_send(
58
- coordinator.processor_method_name,
59
- source
60
- )
56
+ def timeout_reached?
57
+ return false if coordinator.agent_timeout.zero?
61
58
 
62
- dump_to_file(data: processor_result, attacher: :result_dump)
59
+ (created_at + coordinator.agent_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
60
+ end
61
+
62
+ def processing_timeout_reached?
63
+ return false if coordinator.agent_processing_timeout.zero?
63
64
 
64
- finished!
65
+ (started_at + coordinator.agent_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
65
66
  end
66
67
  end
67
68
  end
@@ -8,50 +8,51 @@
8
8
  # agent_queue :string(255) not null
9
9
  # coordinator_queue :string(255) not null
10
10
  # finished_at :datetime
11
- # keep_data :boolean default(FALSE), not null
12
11
  # processor_class_name :string(255) not null
13
12
  # processor_method_name :string(255) not null
14
13
  # size :integer not null
15
14
  # started_at :datetime
16
15
  # state :string(255) not null
17
- # want_result :boolean default(TRUE), not null
18
16
  # created_at :datetime not null
19
17
  # updated_at :datetime not null
20
18
  #
21
19
 
22
20
  module Dipa
23
21
  class Coordinator < ApplicationRecord
24
- include Models::Dipa::Dumpable
25
- include Models::Dipa::Loadable
26
- include Models::Dipa::StateAttributeHandling
22
+ include Dipa::Models::Dumpable
23
+ include Dipa::Models::Loadable
24
+ include Dipa::Models::CoordinatorStateAttributes
25
+ include Dipa::Models::StateAttributeHandling
27
26
 
28
27
  # validation and default for `state` attribute is included by
29
- # Models::Dipa::StateAttributeHandling
30
-
31
- attribute :keep_data, :boolean, default: false
32
- attribute :want_result, :boolean, default: true
28
+ # Dipa::Models::StateAttributeHandling
33
29
 
34
30
  validates :agent_queue, presence: true
35
31
  validates :coordinator_queue, presence: true
36
32
 
37
33
  validates :size, numericality: { only_integer: true }
38
-
39
34
  validates :started_at, 'dipa/date' => true, allow_nil: true
40
35
  validates :finished_at, 'dipa/date' => true, allow_nil: true
41
-
42
36
  validates :processor_class_name, presence: true
43
37
  validates :processor_method_name, presence: true
44
38
 
45
- validates :keep_data, inclusion: [true, false]
46
- validates :want_result, inclusion: [true, false]
47
-
48
39
  has_many :agents, dependent: :destroy, inverse_of: :coordinator,
49
40
  foreign_key: :dipa_coordinator_id
50
41
 
42
+ has_one :coordinator_options_record, dependent: :destroy,
43
+ inverse_of: :coordinator,
44
+ foreign_key: :dipa_coordinator_id
45
+ accepts_nested_attributes_for :coordinator_options_record
46
+
51
47
  has_one_attached :source_dump
52
48
 
49
+ delegate :keep_data?, :want_result?,
50
+ :agent_timeout, :agent_processing_timeout,
51
+ :coordinator_timeout, :coordinator_processing_timeout,
52
+ to: :coordinator_options_record
53
+
53
54
  def result
54
- return unless processed?
55
+ return unless finished?
55
56
 
56
57
  _result_from_agents
57
58
  end
@@ -60,8 +61,20 @@ module Dipa
60
61
  load_from_file(attacher: :source_dump)
61
62
  end
62
63
 
63
- def all_agents_created_and_processed?
64
- _all_agents_created? && _all_agents_processed?
64
+ def all_agents_created_and_finished?
65
+ _all_agents_created? && _all_agents_finished?
66
+ end
67
+
68
+ def timeout_reached?
69
+ return false if coordinator_timeout.zero?
70
+
71
+ (created_at + coordinator_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
72
+ end
73
+
74
+ def processing_timeout_reached?
75
+ return false if coordinator_processing_timeout.zero?
76
+
77
+ (started_at + coordinator_processing_timeout) < Time.current.floor(Dipa::DATETIME_PRECISION)
65
78
  end
66
79
 
67
80
  private
@@ -70,8 +83,8 @@ module Dipa
70
83
  agents.with_attached_result_dump.order(:index).map(&:result)
71
84
  end
72
85
 
73
- def _all_agents_processed?
74
- agents.all?(&:processed?)
86
+ def _all_agents_finished?
87
+ agents.all?(&:finished_state?)
75
88
  end
76
89
 
77
90
  def _all_agents_created?
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ # == Schema Information
4
+ #
5
+ # Table name: dipa_coordinator_options_records
6
+ #
7
+ # id :bigint not null, primary key
8
+ # agent_processing_timeout :integer not null
9
+ # agent_timeout :integer not null
10
+ # coordinator_processing_timeout :integer not null
11
+ # coordinator_timeout :integer not null
12
+ # keep_data :boolean default(FALSE), not null
13
+ # want_result :boolean default(TRUE), not null
14
+ # created_at :datetime not null
15
+ # updated_at :datetime not null
16
+ # dipa_coordinator_id :bigint not null
17
+ #
18
+ # Indexes
19
+ #
20
+ # index_dipa_coordinator_options_records_on_dipa_coordinator_id (dipa_coordinator_id)
21
+ #
22
+ # Foreign Keys
23
+ #
24
+ # fk_rails_... (dipa_coordinator_id => dipa_coordinators.id)
25
+ #
26
+
27
+ module Dipa
28
+ class CoordinatorOptionsRecord < ApplicationRecord
29
+ attribute :keep_data, :boolean, default: false
30
+ attribute :want_result, :boolean, default: true
31
+
32
+ attribute :agent_timeout, :integer
33
+ attribute :agent_processing_timeout, :integer
34
+ attribute :coordinator_timeout, :integer
35
+ attribute :coordinator_processing_timeout, :integer
36
+
37
+ validates :keep_data, inclusion: [true, false]
38
+ validates :want_result, inclusion: [true, false]
39
+
40
+ validates :agent_timeout,
41
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
42
+ validates :agent_processing_timeout,
43
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
44
+ validates :coordinator_timeout,
45
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
46
+ validates :coordinator_processing_timeout,
47
+ numericality: { only_integer: true, greater_than_or_equal_to: 0 }
48
+
49
+ belongs_to :coordinator, inverse_of: :coordinator_options_record,
50
+ foreign_key: :dipa_coordinator_id
51
+ end
52
+ end