aeternitas 0.2.0 → 2.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.gitattributes +1 -0
  3. data/.github/workflows/lint.yml +25 -0
  4. data/.github/workflows/tests.yml +28 -0
  5. data/.gitignore +2 -5
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +35 -0
  8. data/Gemfile +1 -1
  9. data/README.md +93 -148
  10. data/Rakefile +1 -1
  11. data/aeternitas.gemspec +23 -34
  12. data/lib/aeternitas/aeternitas_job.rb +7 -0
  13. data/lib/aeternitas/cleanup_old_metrics_job.rb +12 -0
  14. data/lib/aeternitas/cleanup_stale_locks_job.rb +12 -0
  15. data/lib/aeternitas/errors.rb +1 -2
  16. data/lib/aeternitas/guard.rb +72 -106
  17. data/lib/aeternitas/guard_lock.rb +19 -0
  18. data/lib/aeternitas/maintenance.rb +35 -0
  19. data/lib/aeternitas/metric.rb +12 -0
  20. data/lib/aeternitas/metrics.rb +54 -136
  21. data/lib/aeternitas/poll_job.rb +135 -0
  22. data/lib/aeternitas/pollable/configuration.rb +21 -22
  23. data/lib/aeternitas/pollable/dsl.rb +16 -17
  24. data/lib/aeternitas/pollable.rb +19 -18
  25. data/lib/aeternitas/pollable_meta_data.rb +18 -9
  26. data/lib/aeternitas/polling_frequency.rb +4 -4
  27. data/lib/aeternitas/source.rb +5 -5
  28. data/lib/aeternitas/storage_adapter/file.rb +9 -12
  29. data/lib/aeternitas/storage_adapter.rb +1 -3
  30. data/lib/aeternitas/unique_job_lock.rb +15 -0
  31. data/lib/aeternitas/version.rb +1 -1
  32. data/lib/aeternitas.rb +23 -26
  33. data/lib/generators/aeternitas/install_generator.rb +14 -8
  34. data/lib/generators/aeternitas/templates/add_aeternitas.rb.erb +34 -2
  35. data/lib/generators/aeternitas/templates/initializer.rb +10 -7
  36. metadata +35 -123
  37. data/.idea/.rakeTasks +0 -7
  38. data/.idea/misc.xml +0 -4
  39. data/.idea/modules.xml +0 -8
  40. data/.idea/vcs.xml +0 -6
  41. data/.rspec +0 -2
  42. data/.rubocop.yml +0 -2
  43. data/.travis.yml +0 -8
  44. data/lib/aeternitas/metrics/counter.rb +0 -18
  45. data/lib/aeternitas/metrics/ratio.rb +0 -67
  46. data/lib/aeternitas/metrics/ten_minutes_resolution.rb +0 -40
  47. data/lib/aeternitas/metrics/values.rb +0 -18
  48. data/lib/aeternitas/sidekiq/middleware.rb +0 -31
  49. data/lib/aeternitas/sidekiq/poll_job.rb +0 -30
  50. data/lib/aeternitas/sidekiq.rb +0 -5
  51. data/logo.png +0 -0
  52. data/logo.svg +0 -198
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2664ce79ca3661782669c1368fc5fb8c96aff125427ba33041b79c4cd2ce6eb7
4
- data.tar.gz: 5027b2dffa4674e800053460d98bef952f26daaea472e616f7fd32a04f994e81
3
+ metadata.gz: 2c1d90d4cf7395ae79d96e09c0e3ba2ff163d96a31c3283eef8ec516ca2e2645
4
+ data.tar.gz: 1a245a59727f923ce5e1054e7c043c684f7d14e9b37f4ec65265b30843ff41b1
5
5
  SHA512:
6
- metadata.gz: c2b538976ad2a42689a43d835830f047ece22b93f47dfdd1ca8edc144a7661656a310e2b36fe07c786aac8260d8d1e23052ee137c4bc1d3a43f3d91b2ef559b6
7
- data.tar.gz: 3a0b33ff9da21a20249ab510389d4dd402191dfdb170ec17e11f06747c0e03ef78af7c04da81d7bcf09256c585d52a85ed36c6d00d9d5c6f799418bcdeb62f27
6
+ metadata.gz: 8764240e6d8e929f4ed351886daf38df7a3d76bb09b41631ee344be52cb8322b600eec8750aeb20b2cb3f3397c819b3b3fdf930dc47641aca9bc57a97597bc80
7
+ data.tar.gz: fd450bbf6f00ea6341987974741809fcf0a78c6ef7909ac29af9aeae8c87a603e5d36824748bef7f2cce01d9658b14f8d3222aa8d41a3fb5b2f8b70313c9e4bd
data/.gitattributes ADDED
@@ -0,0 +1 @@
1
+ * text=auto eol=lf
@@ -0,0 +1,25 @@
1
+ name: Lint
2
+
3
+ on:
4
+ push:
5
+ branches: [ "main" ]
6
+ pull_request:
7
+ branches: [ "main" ]
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ rspec:
14
+
15
+ runs-on: ubuntu-latest
16
+
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - name: Set up Ruby
20
+ uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: 3.2
23
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
24
+ - name: Code Style Check
25
+ run: bundle exec standardrb --no-fix
@@ -0,0 +1,28 @@
1
+ name: Tests
2
+
3
+ on:
4
+ push:
5
+ branches: [ "main" ]
6
+ pull_request:
7
+ branches: [ "main" ]
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ rspec:
14
+
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ matrix:
18
+ ruby-version: ['3.2']
19
+
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+ - name: Set up Ruby ${{ matrix.ruby-version }}
23
+ uses: ruby/setup-ruby@v1
24
+ with:
25
+ ruby-version: ${{ matrix.ruby-version }}
26
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
27
+ - name: Run tests
28
+ run: bundle exec rake
data/.gitignore CHANGED
@@ -7,8 +7,5 @@
7
7
  /pkg/
8
8
  /spec/reports/
9
9
  /tmp/
10
-
11
- .idea/
12
-
13
- ## File-based project format:
14
- *.iws
10
+ /db/test.sqlite3*
11
+ *.gem
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.5.7
1
+ 3.2.8
data/CHANGELOG.md ADDED
@@ -0,0 +1,35 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [2.0.0] - Unreleased
11
+
12
+ This is a major rewrite of Æternitas with the primary goals of removing the Redis and Sidekiq dependencies and simplifying the core functionality.
13
+
14
+ ### Changed
15
+ - Updated Core Dependencies: The gem now requires Ruby 3.1+ and ActiveRecord/ActiveJob `>= 7.0`.
16
+ - Complete Backend Overhaul: Æternitas no longer depends on Redis or Sidekiq. It now uses a pure ActiveRecord and ActiveJob backend.
17
+ - Job Uniqueness: Replaced the `sidekiq-unique-jobs` dependency with a built-in, database-backed uniqueness mechanism (`Aeternitas::UniqueJobLock`) to ensure only one `PollJob` per pollable can be enqueued at a time.
18
+ - Job Processing: Replaced Sidekiq-specific workers with a backend-agnostic `Aeternitas::PollJob` that works with any ActiveJob adapter (e.g., SolidQueue, GoodJob).
19
+ - Locking Mechanism: Replaced the Redis-backed Guard with a robust, database-backed distributed lock (`Aeternitas::GuardLock`) using pessimistic locking to prevent race conditions.
20
+ - Metrics System: Replaced the complex, Redis-based `tabstabs` metrics with a simple, database-backed system (`Aeternitas::Metric`). Metrics are now disabled by default.
21
+ - Default Source Storage Path: The default directory for the file storage adapter was changed to `storage/aeternitas/` within a Rails application.
22
+
23
+ ### Added
24
+ - Thundering Herd Prevention: The `PollJob` now intelligently staggers retries when a `GuardIsLocked` error occurs, preventing many jobs from retrying simultaneously and overwhelming a resource.
25
+ - Configurable Metrics: Added `Aeternitas.config.metrics_enabled` and `Aeternitas.config.metric_retention_period` to give users control over metrics collection and data retention.
26
+ - Built-in Maintenance Jobs: Added `Aeternitas::CleanupStaleLocksJob` and `Aeternitas::CleanupOldMetricsJob` to provide a clear, easy way to schedule necessary database cleanup.
27
+
28
+ ### Removed
29
+ - Removed direct gem dependencies on `sidekiq`, `sidekiq-unique-jobs`, `redis`, `connection_pool`, and `tabstabs`.
30
+ - Removed the Sidekiq-specific middleware for handling `GuardIsLocked` errors.
31
+ - Removed the complex, multi-resolution time-series logic from the metrics system.
32
+
33
+ ## [0.2.0 and older] - See legacy repository
34
+ - Initial versions of Æternitas: https://github.com/FHG-IMW/aeternitas
35
+ - Relied on a Sidekiq and Redis backend for job processing, locking, and metrics.
data/Gemfile CHANGED
@@ -1,4 +1,4 @@
1
- source 'https://rubygems.org'
1
+ source "https://rubygems.org"
2
2
 
3
3
  # Specify your gem's dependencies in aeternitas.gemspec
4
4
  gemspec
data/README.md CHANGED
@@ -1,16 +1,16 @@
1
- ![Aeternitas](https://github.com/FHG-IMW/aeternitas/blob/master/logo.png?raw=true)
1
+ # Æternitas
2
2
 
3
- [![Build Status](https://travis-ci.org/FHG-IMW/aeternitas.svg?branch=master)](https://travis-ci.org/FHG-IMW/aeternitas)
3
+ [![Tests](https://github.com/Dietech-Group/aeternitas/actions/workflows/tests.yml/badge.svg)](https://github.com/Dietech-Group/aeternitas/actions/workflows/tests.yml)
4
+ [![Lint](https://github.com/Dietech-Group/aeternitas/actions/workflows/lint.yml/badge.svg)](https://github.com/Dietech-Group/aeternitas/actions/workflows/lint.yml)
4
5
 
5
- A ruby gem for continuous source retrieval and data integration.
6
+ A Ruby gem for continuous source retrieval and data integration.
6
7
 
7
- Aeternitas provides means to regularly "poll" resources (i.e. a website, twitter feed or API) and to permanently store retrieved results.
8
- By default it avoids putting too much load on external servers and stores raw results as compressed files on disk.
9
- Aeternitas can be configured to a wide variety of polling strategies (e.g. frequencies, cooldown periods, ignoring exceptions, deactivating resources, ...).
8
+ Æternitas provides means to regularly "poll" resources (i.e. a website, twitter feed or API) and to permanently store the retrieved results.
9
+ By default, it avoids putting too much load on external servers and stores raw results as compressed files on disk.
10
+ Aeternitas can be configured to a wide variety of polling strategies (e.g. frequencies, cooldown periods, error handling, deactivation on failure).
10
11
 
11
- Aeternitas is meant to be included in a rails application and expects a working sidekiq/redis setup and any kind of database backend.
12
- All meta-data is stored in two database tables (aeternitas_pollable_meta_data and aeternitas_sources) while metrics are stored in Redis
13
- and raw data as compressed files on disk.
12
+ Æternitas is meant to be included in a Rails application and uses a pure ActiveJob and ActiveRecord backend.
13
+ All metadata, locks, and metrics are stored in your application's database, while raw source data is stored as compressed files on disk by default.
14
14
 
15
15
  ## Installation
16
16
 
@@ -22,22 +22,32 @@ gem 'aeternitas'
22
22
 
23
23
  And then execute:
24
24
 
25
- $ bundle install
26
- $ rails generate aeternitas:install
27
- $ rake db:migrate
25
+ ```bash
26
+ $ bundle install
27
+ $ rails generate aeternitas:install
28
+ $ rails db:migrate
29
+ ```
28
30
 
29
- to install gem and generate tables and initializers needed for aeternitas.
31
+ This will install the gem, generate the necessary database tables, and create a configuration initializer.
30
32
 
31
- ## Quickstart
33
+ ### Maintenance
34
+
35
+ Æternitas creates lock and metric records in your database. To prevent this data from growing indefinitely, you should schedule periodic cleanup jobs. The two key maintenance jobs are:
32
36
 
33
- Aeternitas expects you wanting to store single pollables as ActiveRecord Objects. For instance you might want to
34
- monitor several Websites for the usage of the word æternitas and store the websites old states for later analysis.
35
- Using Aeternitas you would create your website model and tables
37
+ - **`Aeternitas::CleanupStaleLocksJob`**: Removes old, expired lock records from crashed workers.
38
+ - **`Aeternitas::CleanupOldMetricsJob`**: Prunes metric data older than the configured `metric_retention_period`.
39
+
40
+ You should schedule these jobs to run periodically (e.g. weekly).
41
+
42
+ ## Quickstart
36
43
 
37
- $ rails generate model Website url:string aeternitas_word_count:integer
44
+ Let's say you want to monitor several websites for the usage of a keyword, e.g. 'aeternitas'. First, create your model:
38
45
 
39
- And then include Aeternitas
46
+ ```bash
47
+ $ rails generate model Website url:string keyword_count:integer
48
+ ```
40
49
 
50
+ Then, include `Aeternitas::Pollable` in your model and define your polling logic.
41
51
 
42
52
  ```ruby
43
53
  class Website < ApplicationRecord
@@ -46,61 +56,49 @@ class Website < ApplicationRecord
46
56
  polling_options do
47
57
  polling_frequency :weekly
48
58
  end
49
- end
50
- ```
51
59
 
52
- For now we are satisfied with aeternitas default setting [TODO:link] except for the polling_frequency. We only want to
53
- check once a week.
54
- Next up we have to implement the websites `poll` method.
55
-
56
- ```ruby
57
60
  def poll
58
61
  page_content = Net::HTTP.get(URI.parse(self.url))
59
- add_source(page_content) #store the retrieved page permanently
60
- aeternitas_word_count = page_content.scan('aeternitas').size
61
- update(aeternitas_word_count: aeternitas_word_count)
62
+ add_source(page_content) # Store the retrieved page content permanently
63
+ count = page_content.scan('aeternitas').size
64
+ update(keyword_count: count)
62
65
  end
66
+ end
63
67
  ```
64
68
 
65
- The poll method is called every time a pollable is good to go. In our example this would be once a week. The time at which
66
- aeternitas will execute the poll method is determined by the pollable metadata stored in a separate table and may be
67
- checked using the `next_polling` method on a website (note: there are several advanced error states which may or may not
68
- allow a pollable to be polled).
69
+ The `poll` method is called each time Æternitas processes the job for this resource. In our example, this would be once a week.
69
70
 
70
- Assuming you have already setup sidekiq the only thing left is to regularly run `Aeternitas.enqueue_due_pollables`
71
- and have a worker consuming the "polling" queue.
71
+ To start the polling process, you need to regularly run `Aeternitas.enqueue_due_pollables` and have an ActiveJob backend (like SolidQueue, GoodJob, etc.) running to process the jobs.
72
72
 
73
- In most cases it makes sense to store polling results as sources to allow further work to be done in separate jobs.
74
- In above example we already added the `page_content`as a source to the website. Aeternitas thereby only stores a new source
75
- if the sources fingerprint does not yet exist (i.e. MD5 Hash of the page_content). If we wanted to process the word count in
76
- a separate job the following implementation would allow to do so.
73
+ In most cases it makes sense to store polling results as sources to allow further work to be done in separate jobs. In above example we already added the `page_content` as a source to the website with `add_source`.
74
+ Aeternitas only stores a new source if the source's fingerprint (MD5 Hash of the content) does not exist yet. If we wanted to process the word count in a separate job the following implementation would allow to do so:
77
75
 
78
76
  ```ruby
77
+ # app/models/website.rb
79
78
  class Website < ApplicationRecord
80
79
  include Aeternitas::Pollable
81
80
 
82
81
  polling_options do
83
82
  polling_frequency :weekly
84
83
  end
85
-
84
+
86
85
  def poll
87
86
  page_content = Net::HTTP.get(URI.parse(self.url))
88
- new_source = add_source(page_content) #returns nil if source already exists
89
- CountWordJob.perform_async(new_source.id) if new_source
90
-
91
-
87
+ new_source = add_source(page_content) # returns nil if source already exists
88
+ CountKeywordJob.perform_later(new_source.id) if new_source
92
89
  end
93
90
  end
94
91
 
95
- class CountWordJob
96
- include Sidekiq::Worker
97
-
92
+ # app/jobs/count_keyword_job.rb
93
+ class CountKeywordJob < ApplicationJob
94
+ queue_as :default
95
+
98
96
  def perform(source_id)
99
97
  source = Aeternitas::Source.find(source_id)
100
98
  page_content = source.raw_content
101
- aeternitas_word_count = page_content.scan('aeternitas').size
99
+ keyword_count = page_content.scan('aeternitas').size
102
100
  website = source.pollable
103
- website.update(aeternitas_word_count: aeternitas_word_count)
101
+ website.update(keyword_count: keyword_count)
104
102
  end
105
103
  end
106
104
  ```
@@ -109,57 +107,45 @@ end
109
107
 
110
108
  ### Global Configuration
111
109
 
112
- In this configuration you can specify the global settings for Æternitas. The configuration should be stored in
113
- `config/initializers/aeternitas.rb`
114
-
115
- #### redis
110
+ Global settings can be configured in `config/initializers/aeternitas.rb`.
116
111
 
117
- This option specifies the redis connection details. Æternitas uses redis to for resource locking and to store statistics.
112
+ #### Metrics
113
+ You can enable or disable metrics collection. By default, metrics are disabled.
118
114
 
119
115
  ```ruby
120
- Aeternitas.configure do |config|
121
- config.redis = {host: localhost, port: 6379}
122
- end
123
- ```
124
-
125
- For configuration options you can have a look here: [redis-rb](https://github.com/redis/redis-rb)
126
-
127
- #### storage_adapter
128
- _Default: Aeternitas::StorageAdapter::File_
116
+ Aeternitas.configure do |config|
117
+ # Set to true to enable logging metrics to the database.
118
+ config.metrics_enabled = true
129
119
 
130
- Æternitas by default stores source file in compressed files on disk. If you however want to store them in another way you
131
- can do so easily by implementing the `Aeternitas::StorageAdapter` interface. For an example you can have a look at
132
- `Aeternitas::StorageAdapter::File`.
133
- To specify which storage adapter Æternitas should use, just pass the class name to this option:
134
-
135
- ```ruby
136
- Aeternitas.configure do |config|
137
- config.storage_adapter = Aeternitas::StorageAdapter::File
120
+ # Configure how long to keep metric data.
121
+ config.metric_retention_period = 180.days
138
122
  end
139
123
  ```
140
124
 
141
- #### storage_adapter_options
142
-
143
- Some storage adapter need some extra configuration. The file adapter for example needs to now where to store the files:
125
+ #### Storage Adapter
126
+ By default, Æternitas stores source files as compressed files on disk. You can change this behavior by implementing a custom storage adapter. For an example you can have a look at `Aeternitas::StorageAdapter::File`.
144
127
 
145
128
  ```ruby
146
- Aeternitas.configure do |config|
147
- config.storage_adapter_options = {
129
+ Aeternitas.configure do |config|
130
+ # To change the storage directory for the default File adapter:
131
+ config.storage_adapter_config = {
148
132
  directory: File.join(Rails.root, 'public', 'sources')
149
133
  }
134
+
135
+ # To use a custom adapter:
136
+ config.storage_adapter = Aeternitas::StorageAdapter::MyCustomAdapter
150
137
  end
151
138
  ```
152
139
 
153
140
  ### Pollable Configuration
154
141
 
155
- Pollables can be configured on a per class base using the `polling_options` block.
156
-
142
+ Pollables can be configured on a per-model basis using the `polling_options` block.
157
143
 
158
144
  #### polling_frequency
159
145
  _Default: :daily_
160
146
 
161
147
  This option controls how often a pollable is polled and can be configured in two different ways.
162
- Either use one of the presets specified in `Aeternitas::PollingFrequency` by specifiey the presets name as a symbol
148
+ Either use one of the presets specified in `Aeternitas::PollingFrequency` by specifying the presets name as a symbol:
163
149
 
164
150
  ```ruby
165
151
  polling_options do
@@ -167,111 +153,67 @@ polling_options do
167
153
  end
168
154
  ```
169
155
 
170
- If want to specify a more complex polling schema you can also us a custom method which returns the time and date of the
171
- next polling. For example if you want to increase the frequency depending on the pollables age you could use the
172
- following method:
156
+ Or, if you want to specify a more complex polling schema you can do so by using a custom lambda for dynamic frequency:
173
157
 
174
158
  ```ruby
175
159
  polling_options do
176
160
  # set frequency depending elements age (+ 1 month for every 3 months)
177
- polling_frequency ->(context) { 1.month + (Time.now - context.created_at).to_i / 3.month * 1.month }
178
- end
179
- ```
180
-
181
- #### before_polling
182
- _Default: []_
183
-
184
- Specifies methods that are run before the polling is executed. You can either specify a method name or a lamba
185
- ```ruby
186
- polling_options do
187
- # run the pollables `foo` and `bar` methods
188
- before_polling :foo, :bar
189
- # run a custom block
190
- before_polling ->(pollable) { puts "About to poll #{pollable.id}"}
161
+ polling_frequency ->(context) { 1.month.from_now + (Time.now - context.created_at).to_i / 3.months * 1.month }
191
162
  end
192
163
  ```
193
164
 
194
- #### after_polling
195
-
196
- Specify methods run after polling was successful. See __before_polling__
197
-
198
- #### deactivation_errors
165
+ #### before_polling / after_polling
199
166
  _Default: []_
200
167
 
201
- Specify error clases which, once they occur, will instantly deactivate the pollable. This can be useful for example if
202
- the error implied that the resource does not exist any more.
168
+ Specify methods to run before each poll or after each successful poll. You can either specify a method name or a lambda:
203
169
 
204
170
  ```ruby
205
171
  polling_options do
206
- # deactivate the pollable if the tweet was not found
207
- deactivation_errors Twitter::Error::NotFound
172
+ before_polling :log_start
173
+ after_polling ->(pollable) { puts "Finished polling #{pollable.id}" }
208
174
  end
209
175
  ```
210
176
 
211
- #### ignore_error
177
+ #### deactivate_on / ignore_error
212
178
  _Default: []_
213
179
 
214
- Errors specified as ignored errors are wrapped within `Aeternitas::Errors::Ignored`. which is then raised instead.
215
- This is supposed to be used in combination with error tracking systems like Airbrake. Instead globally telling Airbrake which errors to
216
- ignore, you can do this on a per pollable basis
180
+ Define custom error handling rules.
181
+
182
+ `deactivate_on` will stop polling a resource permanently if a specified error occurs. This can be useful if the error implied that the resource does not exist anymore.
183
+
184
+ `ignore_error` will wrap the error within `Aeternitas::Errors::Ignored` which is then raised instead. This can be useful for filtering in exception tracking services like Airbrake.
217
185
 
218
186
  ```ruby
219
187
  polling_options do
220
- # don't log an error if the twitter api is down
188
+ deactivate_on Twitter::Error::NotFound
221
189
  ignore_error Twitter::Error::ServiceUnavailable
222
190
  end
223
191
  ```
224
192
 
225
- #### sleep_on_guard_lock
226
- _Default: true
193
+ #### sleep_on_guard_locked
194
+ _Default: false_
195
+
196
+ Controls behavior when a guard lock cannot be acquired.
197
+ - **`false`:** The job will be retried with a smart, staggered backoff delay to prevent a "thundering herd." This is the recommended and most scalable option.
198
+ - **`true`:** The job will cause the ActiveJob worker thread to `sleep` until the lock is expected to be free, blocking that thread from processing other jobs. This is an aggressive strategy and should *only* be used in specific cases where you intend to pause a dedicated worker.
227
199
 
228
- With this option set to true, if a pollable can't acquire the lock, it will sleep until the guard_timeout expires,
229
- effectively blocking the Sidekiq queue from processing any other jobs.
230
- This should *only* be used, if you know that all the jobs within this queue will try to access the same resource and you want to pause the entire queue.
231
-
232
200
  #### queue
233
201
  _Default: 'polling'_
234
202
 
235
- This option specifies the Sidekiq queue into which the poll job will be enqueued.
203
+ This option specifies the ActiveJob queue for the poll job.
236
204
 
237
205
  #### guard_key
238
- _Default: "#{obj.class.name}"_
206
+ _Default: obj.class.name.to_s_
239
207
 
240
- This option specifies the guard key. This can be done by either specifying a method name or a custom block.
241
- Default is to lock on pollable class level. Therefor only one job at a time per pollable class will be executed to
242
- avoid DDOSing by accident.
208
+ Defines the key used for resource locking. By default, all instances of a model share the same lock. The default is to lock on pollable class level, but you can also provide a block for more granular locking (e.g. per-instance or per-API-host):
243
209
 
244
210
  ```ruby
245
211
  polling_options do
246
- # use the urls host as lock key
247
- guard ->(website) { URI.parse(website.url).host }
212
+ # Lock based on the instance's URL host
213
+ guard_key ->(website) { URI.parse(website.url).host }
248
214
  end
249
215
  ```
250
216
 
251
- ## Web UI
252
-
253
- Æternitas also has a monitoring interface which can be integrated with a rail application.
254
-
255
- To use it add this line to your application's Gemfile:
256
-
257
- ```ruby
258
- gem 'aeternitas_web_ui'
259
- ```
260
-
261
- Then run
262
-
263
- $ bundle install
264
-
265
- And mount the engine
266
-
267
- ```ruby
268
- # config/routes.rb
269
-
270
- mount Aeternitas::WebUi::Engine => '/aeternitas'
271
- ```
272
-
273
- For more information head over to the project page: [https://github.com/FHG-IMW/aeternitas_web_ui](https://github.com/FHG-IMW/aeternitas_web_ui)
274
-
275
217
  ## Development
276
218
 
277
219
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -280,10 +222,13 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
280
222
 
281
223
  ## Contributing
282
224
 
283
- Bug reports and spec backed pull requests are welcome on GitHub at https://github.com/FHG-IMW/aeternitas. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
225
+ Bug reports and spec backed pull requests are welcome on GitHub at https://github.com/Dietech-Group/aeternitas. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
284
226
 
285
227
 
286
228
  ## License
287
229
 
288
230
  The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
289
231
 
232
+ ## History
233
+
234
+ This gem was originally developed under [FHG-IMW/aeternitas](https://github.com/FHG-IMW/aeternitas) and named "æternitas - A ruby gem for continuous source retrieval and data integration". It's core was based upon Sidekiq and Redis which both were removed as dependencies for this gem.
data/Rakefile CHANGED
@@ -3,4 +3,4 @@ require "rspec/core/rake_task"
3
3
 
4
4
  RSpec::Core::RakeTask.new(:spec)
5
5
 
6
- task :default => :spec
6
+ task default: :spec
data/aeternitas.gemspec CHANGED
@@ -1,44 +1,33 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
1
+ lib = File.expand_path("../lib", __FILE__)
3
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'aeternitas/version'
3
+ require "aeternitas/version"
5
4
 
6
5
  Gem::Specification.new do |spec|
7
- spec.name = 'aeternitas'
8
- spec.version = Aeternitas::VERSION
9
- spec.authors = ['Michael Prilop', 'Max Kießling', 'Robert Terbach']
10
- spec.email = ['max@kopfueber.org', 'michael.prilop@imw.fraunhofer.de']
6
+ spec.name = "aeternitas"
7
+ spec.version = Aeternitas::VERSION
8
+ spec.authors = ["Michael Prilop", "Max Kießling", "Louis Franzke"]
9
+ spec.email = ["prilop@infai.org"]
11
10
 
12
- spec.summary = "æternitas - A ruby gem for continuous source retrieval and data integration"
13
- spec.description = <<-EOF
14
- Æternitas provides means to regularly 'poll' resources (i.e. a website, twitter feed or API) and to permanently
15
- store retrieved results. By default æternitas avoids putting too much load on external servers and stores raw
16
- results as compressed files on disk. It can be configured to a wide variety of polling strategies (e.g. frequencies,
17
- cooldown periods, ignoring exceptions, deactivating resources, ...)."
18
- EOF
19
- spec.homepage = "https://github.com/FHG-IMW/aeternitas"
20
- spec.license = "MIT"
11
+ spec.summary = "æternitas - version 2"
12
+ spec.description = "æternitas is a tool support polling resources (webpages, APIs)."
13
+ spec.homepage = "https://github.com/Dietech-Group/aeternitas"
14
+ spec.license = "MIT"
21
15
 
22
- spec.files = `git ls-files -z`.split("\x0").reject do |f|
16
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
23
17
  f.match(%r{^(test|spec|features)/})
24
18
  end
25
- spec.bindir = 'exe'
26
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
27
- spec.require_paths = ['lib']
19
+ spec.bindir = "exe"
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ["lib"]
28
22
 
29
- spec.add_dependency 'activerecord', '>= 5'
30
- spec.add_dependency 'redis'
31
- spec.add_dependency 'connection_pool'
32
- spec.add_dependency 'aasm'
33
- spec.add_dependency 'sidekiq', '> 4', '< 6'
34
- spec.add_dependency 'sidekiq-unique-jobs', '~> 5.0'
35
- spec.add_dependency 'tabstabs'
23
+ spec.add_dependency "activerecord", ">= 7.0"
24
+ spec.add_dependency "activejob", ">= 7.0"
25
+ spec.add_dependency "aasm"
36
26
 
37
- spec.add_development_dependency 'bundler'
38
- spec.add_development_dependency 'rake'
39
- spec.add_development_dependency 'rspec', '~> 3.0'
40
- spec.add_development_dependency 'sqlite3'
41
- spec.add_development_dependency 'database_cleaner', '~> 1.5'
42
- spec.add_development_dependency 'rspec-sidekiq'
43
- spec.add_development_dependency 'mock_redis'
27
+ spec.add_development_dependency "bundler"
28
+ spec.add_development_dependency "rake"
29
+ spec.add_development_dependency "rspec-rails", "~> 7.0"
30
+ spec.add_development_dependency "sqlite3", "~> 2.1"
31
+ spec.add_development_dependency "database_cleaner", "~> 2.0"
32
+ spec.add_development_dependency "standard"
44
33
  end
@@ -0,0 +1,7 @@
1
+ require "active_job"
2
+
3
+ module Aeternitas
4
+ # Base class for Aeternitas ActiveJobs
5
+ class AeternitasJob < ActiveJob::Base
6
+ end
7
+ end
@@ -0,0 +1,12 @@
1
+ require_relative "aeternitas_job"
2
+
3
+ module Aeternitas
4
+ # An ActiveJob for cleaning up old metric records.
5
+ class CleanupOldMetricsJob < AeternitasJob
6
+ queue_as :maintenance
7
+
8
+ def perform
9
+ Aeternitas::Maintenance.cleanup_old_metrics
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,12 @@
1
+ require_relative "aeternitas_job"
2
+
3
+ module Aeternitas
4
+ # An ActiveJob for cleaning up stale lock records.
5
+ class CleanupStaleLocksJob < AeternitasJob
6
+ queue_as :maintenance
7
+
8
+ def perform
9
+ Aeternitas::Maintenance.cleanup_stale_locks
10
+ end
11
+ end
12
+ end
@@ -14,7 +14,6 @@ module Aeternitas
14
14
  @original_error = original_error
15
15
  super("#{original_error.class} - #{original_error.message}")
16
16
  end
17
-
18
17
  end
19
18
 
20
19
  # Raised when a source data already exists.
@@ -45,4 +44,4 @@ module Aeternitas
45
44
  end
46
45
  end
47
46
  end
48
- end
47
+ end