aeternitas 0.2.0 → 2.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitattributes +1 -0
- data/.github/workflows/lint.yml +25 -0
- data/.github/workflows/tests.yml +28 -0
- data/.gitignore +2 -5
- data/.ruby-version +1 -1
- data/CHANGELOG.md +35 -0
- data/Gemfile +1 -1
- data/README.md +93 -148
- data/Rakefile +1 -1
- data/aeternitas.gemspec +23 -34
- data/lib/aeternitas/aeternitas_job.rb +7 -0
- data/lib/aeternitas/cleanup_old_metrics_job.rb +12 -0
- data/lib/aeternitas/cleanup_stale_locks_job.rb +12 -0
- data/lib/aeternitas/errors.rb +1 -2
- data/lib/aeternitas/guard.rb +72 -106
- data/lib/aeternitas/guard_lock.rb +19 -0
- data/lib/aeternitas/maintenance.rb +35 -0
- data/lib/aeternitas/metric.rb +12 -0
- data/lib/aeternitas/metrics.rb +54 -136
- data/lib/aeternitas/poll_job.rb +135 -0
- data/lib/aeternitas/pollable/configuration.rb +21 -22
- data/lib/aeternitas/pollable/dsl.rb +16 -17
- data/lib/aeternitas/pollable.rb +19 -18
- data/lib/aeternitas/pollable_meta_data.rb +18 -9
- data/lib/aeternitas/polling_frequency.rb +4 -4
- data/lib/aeternitas/source.rb +5 -5
- data/lib/aeternitas/storage_adapter/file.rb +9 -12
- data/lib/aeternitas/storage_adapter.rb +1 -3
- data/lib/aeternitas/unique_job_lock.rb +15 -0
- data/lib/aeternitas/version.rb +1 -1
- data/lib/aeternitas.rb +23 -26
- data/lib/generators/aeternitas/install_generator.rb +14 -8
- data/lib/generators/aeternitas/templates/add_aeternitas.rb.erb +34 -2
- data/lib/generators/aeternitas/templates/initializer.rb +10 -7
- metadata +35 -123
- data/.idea/.rakeTasks +0 -7
- data/.idea/misc.xml +0 -4
- data/.idea/modules.xml +0 -8
- data/.idea/vcs.xml +0 -6
- data/.rspec +0 -2
- data/.rubocop.yml +0 -2
- data/.travis.yml +0 -8
- data/lib/aeternitas/metrics/counter.rb +0 -18
- data/lib/aeternitas/metrics/ratio.rb +0 -67
- data/lib/aeternitas/metrics/ten_minutes_resolution.rb +0 -40
- data/lib/aeternitas/metrics/values.rb +0 -18
- data/lib/aeternitas/sidekiq/middleware.rb +0 -31
- data/lib/aeternitas/sidekiq/poll_job.rb +0 -30
- data/lib/aeternitas/sidekiq.rb +0 -5
- data/logo.png +0 -0
- data/logo.svg +0 -198
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2c1d90d4cf7395ae79d96e09c0e3ba2ff163d96a31c3283eef8ec516ca2e2645
|
4
|
+
data.tar.gz: 1a245a59727f923ce5e1054e7c043c684f7d14e9b37f4ec65265b30843ff41b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8764240e6d8e929f4ed351886daf38df7a3d76bb09b41631ee344be52cb8322b600eec8750aeb20b2cb3f3397c819b3b3fdf930dc47641aca9bc57a97597bc80
|
7
|
+
data.tar.gz: fd450bbf6f00ea6341987974741809fcf0a78c6ef7909ac29af9aeae8c87a603e5d36824748bef7f2cce01d9658b14f8d3222aa8d41a3fb5b2f8b70313c9e4bd
|
data/.gitattributes
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* text=auto eol=lf
|
@@ -0,0 +1,25 @@
|
|
1
|
+
name: Lint
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ "main" ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ "main" ]
|
8
|
+
|
9
|
+
permissions:
|
10
|
+
contents: read
|
11
|
+
|
12
|
+
jobs:
|
13
|
+
rspec:
|
14
|
+
|
15
|
+
runs-on: ubuntu-latest
|
16
|
+
|
17
|
+
steps:
|
18
|
+
- uses: actions/checkout@v4
|
19
|
+
- name: Set up Ruby
|
20
|
+
uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: 3.2
|
23
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
24
|
+
- name: Code Style Check
|
25
|
+
run: bundle exec standardrb --no-fix
|
@@ -0,0 +1,28 @@
|
|
1
|
+
name: Tests
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ "main" ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ "main" ]
|
8
|
+
|
9
|
+
permissions:
|
10
|
+
contents: read
|
11
|
+
|
12
|
+
jobs:
|
13
|
+
rspec:
|
14
|
+
|
15
|
+
runs-on: ubuntu-latest
|
16
|
+
strategy:
|
17
|
+
matrix:
|
18
|
+
ruby-version: ['3.2']
|
19
|
+
|
20
|
+
steps:
|
21
|
+
- uses: actions/checkout@v4
|
22
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
23
|
+
uses: ruby/setup-ruby@v1
|
24
|
+
with:
|
25
|
+
ruby-version: ${{ matrix.ruby-version }}
|
26
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
27
|
+
- name: Run tests
|
28
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
3.2.8
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
All notable changes to this project will be documented in this file.
|
4
|
+
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
7
|
+
|
8
|
+
## [Unreleased]
|
9
|
+
|
10
|
+
## [2.0.0] - Unreleased
|
11
|
+
|
12
|
+
This is a major rewrite of Æternitas with the primary goals of removing the Redis and Sidekiq dependencies and simplifying the core functionality.
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
- Updated Core Dependencies: The gem now requires Ruby 3.1+ and ActiveRecord/ActiveJob `>= 7.0`.
|
16
|
+
- Complete Backend Overhaul: Æternitas no longer depends on Redis or Sidekiq. It now uses a pure ActiveRecord and ActiveJob backend.
|
17
|
+
- Job Uniqueness: Replaced the `sidekiq-unique-jobs` dependency with a built-in, database-backed uniqueness mechanism (`Aeternitas::UniqueJobLock`) to ensure only one `PollJob` per pollable can be enqueued at a time.
|
18
|
+
- Job Processing: Replaced Sidekiq-specific workers with a backend-agnostic `Aeternitas::PollJob` that works with any ActiveJob adapter (e.g., SolidQueue, GoodJob).
|
19
|
+
- Locking Mechanism: Replaced the Redis-backed Guard with a robust, database-backed distributed lock (`Aeternitas::GuardLock`) using pessimistic locking to prevent race conditions.
|
20
|
+
- Metrics System: Replaced the complex, Redis-based `tabstabs` metrics with a simple, database-backed system (`Aeternitas::Metric`). Metrics are now disabled by default.
|
21
|
+
- Default Source Storage Path: The default directory for the file storage adapter was changed to `storage/aeternitas/` within a Rails application.
|
22
|
+
|
23
|
+
### Added
|
24
|
+
- Thundering Herd Prevention: The `PollJob` now intelligently staggers retries when a `GuardIsLocked` error occurs, preventing many jobs from retrying simultaneously and overwhelming a resource.
|
25
|
+
- Configurable Metrics: Added `Aeternitas.config.metrics_enabled` and `Aeternitas.config.metric_retention_period` to give users control over metrics collection and data retention.
|
26
|
+
- Built-in Maintenance Jobs: Added `Aeternitas::CleanupStaleLocksJob` and `Aeternitas::CleanupOldMetricsJob` to provide a clear, easy way to schedule necessary database cleanup.
|
27
|
+
|
28
|
+
### Removed
|
29
|
+
- Removed direct gem dependencies on `sidekiq`, `sidekiq-unique-jobs`, `redis`, `connection_pool`, and `tabstabs`.
|
30
|
+
- Removed the Sidekiq-specific middleware for handling `GuardIsLocked` errors.
|
31
|
+
- Removed the complex, multi-resolution time-series logic from the metrics system.
|
32
|
+
|
33
|
+
## [0.2.0 and older] - See legacy repository
|
34
|
+
- Initial versions of Æternitas: https://github.com/FHG-IMW/aeternitas
|
35
|
+
- Relied on a Sidekiq and Redis backend for job processing, locking, and metrics.
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,16 +1,16 @@
|
|
1
|
-
|
1
|
+
# Æternitas
|
2
2
|
|
3
|
-
[](https://github.com/Dietech-Group/aeternitas/actions/workflows/tests.yml)
|
4
|
+
[](https://github.com/Dietech-Group/aeternitas/actions/workflows/lint.yml)
|
4
5
|
|
5
|
-
A
|
6
|
+
A Ruby gem for continuous source retrieval and data integration.
|
6
7
|
|
7
|
-
|
8
|
-
By default it avoids putting too much load on external servers and stores raw results as compressed files on disk.
|
9
|
-
Aeternitas can be configured to a wide variety of polling strategies (e.g. frequencies, cooldown periods,
|
8
|
+
Æternitas provides means to regularly "poll" resources (i.e. a website, twitter feed or API) and to permanently store the retrieved results.
|
9
|
+
By default, it avoids putting too much load on external servers and stores raw results as compressed files on disk.
|
10
|
+
Aeternitas can be configured to a wide variety of polling strategies (e.g. frequencies, cooldown periods, error handling, deactivation on failure).
|
10
11
|
|
11
|
-
|
12
|
-
All
|
13
|
-
and raw data as compressed files on disk.
|
12
|
+
Æternitas is meant to be included in a Rails application and uses a pure ActiveJob and ActiveRecord backend.
|
13
|
+
All metadata, locks, and metrics are stored in your application's database, while raw source data is stored as compressed files on disk by default.
|
14
14
|
|
15
15
|
## Installation
|
16
16
|
|
@@ -22,22 +22,32 @@ gem 'aeternitas'
|
|
22
22
|
|
23
23
|
And then execute:
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
```bash
|
26
|
+
$ bundle install
|
27
|
+
$ rails generate aeternitas:install
|
28
|
+
$ rails db:migrate
|
29
|
+
```
|
28
30
|
|
29
|
-
|
31
|
+
This will install the gem, generate the necessary database tables, and create a configuration initializer.
|
30
32
|
|
31
|
-
|
33
|
+
### Maintenance
|
34
|
+
|
35
|
+
Æternitas creates lock and metric records in your database. To prevent this data from growing indefinitely, you should schedule periodic cleanup jobs. The two key maintenance jobs are:
|
32
36
|
|
33
|
-
Aeternitas
|
34
|
-
|
35
|
-
|
37
|
+
- **`Aeternitas::CleanupStaleLocksJob`**: Removes old, expired lock records from crashed workers.
|
38
|
+
- **`Aeternitas::CleanupOldMetricsJob`**: Prunes metric data older than the configured `metric_retention_period`.
|
39
|
+
|
40
|
+
You should schedule these jobs to run periodically (e.g. weekly).
|
41
|
+
|
42
|
+
## Quickstart
|
36
43
|
|
37
|
-
|
44
|
+
Let's say you want to monitor several websites for the usage of a keyword, e.g. 'aeternitas'. First, create your model:
|
38
45
|
|
39
|
-
|
46
|
+
```bash
|
47
|
+
$ rails generate model Website url:string keyword_count:integer
|
48
|
+
```
|
40
49
|
|
50
|
+
Then, include `Aeternitas::Pollable` in your model and define your polling logic.
|
41
51
|
|
42
52
|
```ruby
|
43
53
|
class Website < ApplicationRecord
|
@@ -46,61 +56,49 @@ class Website < ApplicationRecord
|
|
46
56
|
polling_options do
|
47
57
|
polling_frequency :weekly
|
48
58
|
end
|
49
|
-
end
|
50
|
-
```
|
51
59
|
|
52
|
-
For now we are satisfied with aeternitas default setting [TODO:link] except for the polling_frequency. We only want to
|
53
|
-
check once a week.
|
54
|
-
Next up we have to implement the websites `poll` method.
|
55
|
-
|
56
|
-
```ruby
|
57
60
|
def poll
|
58
61
|
page_content = Net::HTTP.get(URI.parse(self.url))
|
59
|
-
add_source(page_content) #
|
60
|
-
|
61
|
-
update(
|
62
|
+
add_source(page_content) # Store the retrieved page content permanently
|
63
|
+
count = page_content.scan('aeternitas').size
|
64
|
+
update(keyword_count: count)
|
62
65
|
end
|
66
|
+
end
|
63
67
|
```
|
64
68
|
|
65
|
-
The poll method is called
|
66
|
-
aeternitas will execute the poll method is determined by the pollable metadata stored in a separate table and may be
|
67
|
-
checked using the `next_polling` method on a website (note: there are several advanced error states which may or may not
|
68
|
-
allow a pollable to be polled).
|
69
|
+
The `poll` method is called each time Æternitas processes the job for this resource. In our example, this would be once a week.
|
69
70
|
|
70
|
-
|
71
|
-
and have a worker consuming the "polling" queue.
|
71
|
+
To start the polling process, you need to regularly run `Aeternitas.enqueue_due_pollables` and have an ActiveJob backend (like SolidQueue, GoodJob, etc.) running to process the jobs.
|
72
72
|
|
73
|
-
In most cases it makes sense to store polling results as sources to allow further work to be done in separate jobs.
|
74
|
-
|
75
|
-
if the sources fingerprint does not yet exist (i.e. MD5 Hash of the page_content). If we wanted to process the word count in
|
76
|
-
a separate job the following implementation would allow to do so.
|
73
|
+
In most cases it makes sense to store polling results as sources to allow further work to be done in separate jobs. In above example we already added the `page_content` as a source to the website with `add_source`.
|
74
|
+
Aeternitas only stores a new source if the source's fingerprint (MD5 Hash of the content) does not exist yet. If we wanted to process the word count in a separate job the following implementation would allow to do so:
|
77
75
|
|
78
76
|
```ruby
|
77
|
+
# app/models/website.rb
|
79
78
|
class Website < ApplicationRecord
|
80
79
|
include Aeternitas::Pollable
|
81
80
|
|
82
81
|
polling_options do
|
83
82
|
polling_frequency :weekly
|
84
83
|
end
|
85
|
-
|
84
|
+
|
86
85
|
def poll
|
87
86
|
page_content = Net::HTTP.get(URI.parse(self.url))
|
88
|
-
new_source = add_source(page_content) #returns nil if source already exists
|
89
|
-
|
90
|
-
|
91
|
-
|
87
|
+
new_source = add_source(page_content) # returns nil if source already exists
|
88
|
+
CountKeywordJob.perform_later(new_source.id) if new_source
|
92
89
|
end
|
93
90
|
end
|
94
91
|
|
95
|
-
|
96
|
-
|
97
|
-
|
92
|
+
# app/jobs/count_keyword_job.rb
|
93
|
+
class CountKeywordJob < ApplicationJob
|
94
|
+
queue_as :default
|
95
|
+
|
98
96
|
def perform(source_id)
|
99
97
|
source = Aeternitas::Source.find(source_id)
|
100
98
|
page_content = source.raw_content
|
101
|
-
|
99
|
+
keyword_count = page_content.scan('aeternitas').size
|
102
100
|
website = source.pollable
|
103
|
-
website.update(
|
101
|
+
website.update(keyword_count: keyword_count)
|
104
102
|
end
|
105
103
|
end
|
106
104
|
```
|
@@ -109,57 +107,45 @@ end
|
|
109
107
|
|
110
108
|
### Global Configuration
|
111
109
|
|
112
|
-
|
113
|
-
`config/initializers/aeternitas.rb`
|
114
|
-
|
115
|
-
#### redis
|
110
|
+
Global settings can be configured in `config/initializers/aeternitas.rb`.
|
116
111
|
|
117
|
-
|
112
|
+
#### Metrics
|
113
|
+
You can enable or disable metrics collection. By default, metrics are disabled.
|
118
114
|
|
119
115
|
```ruby
|
120
|
-
Aeternitas.configure do |config|
|
121
|
-
|
122
|
-
|
123
|
-
```
|
124
|
-
|
125
|
-
For configuration options you can have a look here: [redis-rb](https://github.com/redis/redis-rb)
|
126
|
-
|
127
|
-
#### storage_adapter
|
128
|
-
_Default: Aeternitas::StorageAdapter::File_
|
116
|
+
Aeternitas.configure do |config|
|
117
|
+
# Set to true to enable logging metrics to the database.
|
118
|
+
config.metrics_enabled = true
|
129
119
|
|
130
|
-
|
131
|
-
|
132
|
-
`Aeternitas::StorageAdapter::File`.
|
133
|
-
To specify which storage adapter Æternitas should use, just pass the class name to this option:
|
134
|
-
|
135
|
-
```ruby
|
136
|
-
Aeternitas.configure do |config|
|
137
|
-
config.storage_adapter = Aeternitas::StorageAdapter::File
|
120
|
+
# Configure how long to keep metric data.
|
121
|
+
config.metric_retention_period = 180.days
|
138
122
|
end
|
139
123
|
```
|
140
124
|
|
141
|
-
####
|
142
|
-
|
143
|
-
Some storage adapter need some extra configuration. The file adapter for example needs to now where to store the files:
|
125
|
+
#### Storage Adapter
|
126
|
+
By default, Æternitas stores source files as compressed files on disk. You can change this behavior by implementing a custom storage adapter. For an example you can have a look at `Aeternitas::StorageAdapter::File`.
|
144
127
|
|
145
128
|
```ruby
|
146
|
-
Aeternitas.configure do |config|
|
147
|
-
|
129
|
+
Aeternitas.configure do |config|
|
130
|
+
# To change the storage directory for the default File adapter:
|
131
|
+
config.storage_adapter_config = {
|
148
132
|
directory: File.join(Rails.root, 'public', 'sources')
|
149
133
|
}
|
134
|
+
|
135
|
+
# To use a custom adapter:
|
136
|
+
config.storage_adapter = Aeternitas::StorageAdapter::MyCustomAdapter
|
150
137
|
end
|
151
138
|
```
|
152
139
|
|
153
140
|
### Pollable Configuration
|
154
141
|
|
155
|
-
Pollables can be configured on a per
|
156
|
-
|
142
|
+
Pollables can be configured on a per-model basis using the `polling_options` block.
|
157
143
|
|
158
144
|
#### polling_frequency
|
159
145
|
_Default: :daily_
|
160
146
|
|
161
147
|
This option controls how often a pollable is polled and can be configured in two different ways.
|
162
|
-
Either use one of the presets specified in `Aeternitas::PollingFrequency` by
|
148
|
+
Either use one of the presets specified in `Aeternitas::PollingFrequency` by specifying the presets name as a symbol:
|
163
149
|
|
164
150
|
```ruby
|
165
151
|
polling_options do
|
@@ -167,111 +153,67 @@ polling_options do
|
|
167
153
|
end
|
168
154
|
```
|
169
155
|
|
170
|
-
|
171
|
-
next polling. For example if you want to increase the frequency depending on the pollables age you could use the
|
172
|
-
following method:
|
156
|
+
Or, if you want to specify a more complex polling schema you can do so by using a custom lambda for dynamic frequency:
|
173
157
|
|
174
158
|
```ruby
|
175
159
|
polling_options do
|
176
160
|
# set frequency depending elements age (+ 1 month for every 3 months)
|
177
|
-
polling_frequency ->(context) { 1.month + (Time.now - context.created_at).to_i / 3.
|
178
|
-
end
|
179
|
-
```
|
180
|
-
|
181
|
-
#### before_polling
|
182
|
-
_Default: []_
|
183
|
-
|
184
|
-
Specifies methods that are run before the polling is executed. You can either specify a method name or a lamba
|
185
|
-
```ruby
|
186
|
-
polling_options do
|
187
|
-
# run the pollables `foo` and `bar` methods
|
188
|
-
before_polling :foo, :bar
|
189
|
-
# run a custom block
|
190
|
-
before_polling ->(pollable) { puts "About to poll #{pollable.id}"}
|
161
|
+
polling_frequency ->(context) { 1.month.from_now + (Time.now - context.created_at).to_i / 3.months * 1.month }
|
191
162
|
end
|
192
163
|
```
|
193
164
|
|
194
|
-
#### after_polling
|
195
|
-
|
196
|
-
Specify methods run after polling was successful. See __before_polling__
|
197
|
-
|
198
|
-
#### deactivation_errors
|
165
|
+
#### before_polling / after_polling
|
199
166
|
_Default: []_
|
200
167
|
|
201
|
-
Specify
|
202
|
-
the error implied that the resource does not exist any more.
|
168
|
+
Specify methods to run before each poll or after each successful poll. You can either specify a method name or a lambda:
|
203
169
|
|
204
170
|
```ruby
|
205
171
|
polling_options do
|
206
|
-
|
207
|
-
|
172
|
+
before_polling :log_start
|
173
|
+
after_polling ->(pollable) { puts "Finished polling #{pollable.id}" }
|
208
174
|
end
|
209
175
|
```
|
210
176
|
|
211
|
-
#### ignore_error
|
177
|
+
#### deactivate_on / ignore_error
|
212
178
|
_Default: []_
|
213
179
|
|
214
|
-
|
215
|
-
|
216
|
-
|
180
|
+
Define custom error handling rules.
|
181
|
+
|
182
|
+
`deactivate_on` will stop polling a resource permanently if a specified error occurs. This can be useful if the error implied that the resource does not exist anymore.
|
183
|
+
|
184
|
+
`ignore_error` will wrap the error within `Aeternitas::Errors::Ignored` which is then raised instead. This can be useful for filtering in exception tracking services like Airbrake.
|
217
185
|
|
218
186
|
```ruby
|
219
187
|
polling_options do
|
220
|
-
|
188
|
+
deactivate_on Twitter::Error::NotFound
|
221
189
|
ignore_error Twitter::Error::ServiceUnavailable
|
222
190
|
end
|
223
191
|
```
|
224
192
|
|
225
|
-
####
|
226
|
-
_Default:
|
193
|
+
#### sleep_on_guard_locked
|
194
|
+
_Default: false_
|
195
|
+
|
196
|
+
Controls behavior when a guard lock cannot be acquired.
|
197
|
+
- **`false`:** The job will be retried with a smart, staggered backoff delay to prevent a "thundering herd." This is the recommended and most scalable option.
|
198
|
+
- **`true`:** The job will cause the ActiveJob worker thread to `sleep` until the lock is expected to be free, blocking that thread from processing other jobs. This is an aggressive strategy and should *only* be used in specific cases where you intend to pause a dedicated worker.
|
227
199
|
|
228
|
-
With this option set to true, if a pollable can't acquire the lock, it will sleep until the guard_timeout expires,
|
229
|
-
effectively blocking the Sidekiq queue from processing any other jobs.
|
230
|
-
This should *only* be used, if you know that all the jobs within this queue will try to access the same resource and you want to pause the entire queue.
|
231
|
-
|
232
200
|
#### queue
|
233
201
|
_Default: 'polling'_
|
234
202
|
|
235
|
-
This option specifies the
|
203
|
+
This option specifies the ActiveJob queue for the poll job.
|
236
204
|
|
237
205
|
#### guard_key
|
238
|
-
_Default:
|
206
|
+
_Default: obj.class.name.to_s_
|
239
207
|
|
240
|
-
|
241
|
-
Default is to lock on pollable class level. Therefor only one job at a time per pollable class will be executed to
|
242
|
-
avoid DDOSing by accident.
|
208
|
+
Defines the key used for resource locking. By default, all instances of a model share the same lock. The default is to lock on pollable class level, but you can also provide a block for more granular locking (e.g. per-instance or per-API-host):
|
243
209
|
|
244
210
|
```ruby
|
245
211
|
polling_options do
|
246
|
-
#
|
247
|
-
|
212
|
+
# Lock based on the instance's URL host
|
213
|
+
guard_key ->(website) { URI.parse(website.url).host }
|
248
214
|
end
|
249
215
|
```
|
250
216
|
|
251
|
-
## Web UI
|
252
|
-
|
253
|
-
Æternitas also has a monitoring interface which can be integrated with a rail application.
|
254
|
-
|
255
|
-
To use it add this line to your application's Gemfile:
|
256
|
-
|
257
|
-
```ruby
|
258
|
-
gem 'aeternitas_web_ui'
|
259
|
-
```
|
260
|
-
|
261
|
-
Then run
|
262
|
-
|
263
|
-
$ bundle install
|
264
|
-
|
265
|
-
And mount the engine
|
266
|
-
|
267
|
-
```ruby
|
268
|
-
# config/routes.rb
|
269
|
-
|
270
|
-
mount Aeternitas::WebUi::Engine => '/aeternitas'
|
271
|
-
```
|
272
|
-
|
273
|
-
For more information head over to the project page: [https://github.com/FHG-IMW/aeternitas_web_ui](https://github.com/FHG-IMW/aeternitas_web_ui)
|
274
|
-
|
275
217
|
## Development
|
276
218
|
|
277
219
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -280,10 +222,13 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
280
222
|
|
281
223
|
## Contributing
|
282
224
|
|
283
|
-
Bug reports and spec backed pull requests are welcome on GitHub at https://github.com/
|
225
|
+
Bug reports and spec backed pull requests are welcome on GitHub at https://github.com/Dietech-Group/aeternitas. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
284
226
|
|
285
227
|
|
286
228
|
## License
|
287
229
|
|
288
230
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
289
231
|
|
232
|
+
## History
|
233
|
+
|
234
|
+
This gem was originally developed under [FHG-IMW/aeternitas](https://github.com/FHG-IMW/aeternitas) and named "æternitas - A ruby gem for continuous source retrieval and data integration". It's core was based upon Sidekiq and Redis which both were removed as dependencies for this gem.
|
data/Rakefile
CHANGED
data/aeternitas.gemspec
CHANGED
@@ -1,44 +1,33 @@
|
|
1
|
-
|
2
|
-
lib = File.expand_path('../lib', __FILE__)
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
3
|
+
require "aeternitas/version"
|
5
4
|
|
6
5
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.authors
|
10
|
-
spec.email
|
6
|
+
spec.name = "aeternitas"
|
7
|
+
spec.version = Aeternitas::VERSION
|
8
|
+
spec.authors = ["Michael Prilop", "Max Kießling", "Louis Franzke"]
|
9
|
+
spec.email = ["prilop@infai.org"]
|
11
10
|
|
12
|
-
spec.summary
|
13
|
-
spec.description
|
14
|
-
|
15
|
-
|
16
|
-
results as compressed files on disk. It can be configured to a wide variety of polling strategies (e.g. frequencies,
|
17
|
-
cooldown periods, ignoring exceptions, deactivating resources, ...)."
|
18
|
-
EOF
|
19
|
-
spec.homepage = "https://github.com/FHG-IMW/aeternitas"
|
20
|
-
spec.license = "MIT"
|
11
|
+
spec.summary = "æternitas - version 2"
|
12
|
+
spec.description = "æternitas is a tool support polling resources (webpages, APIs)."
|
13
|
+
spec.homepage = "https://github.com/Dietech-Group/aeternitas"
|
14
|
+
spec.license = "MIT"
|
21
15
|
|
22
|
-
spec.files
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
23
17
|
f.match(%r{^(test|spec|features)/})
|
24
18
|
end
|
25
|
-
spec.bindir
|
26
|
-
spec.executables
|
27
|
-
spec.require_paths = [
|
19
|
+
spec.bindir = "exe"
|
20
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
+
spec.require_paths = ["lib"]
|
28
22
|
|
29
|
-
spec.add_dependency
|
30
|
-
spec.add_dependency
|
31
|
-
spec.add_dependency
|
32
|
-
spec.add_dependency 'aasm'
|
33
|
-
spec.add_dependency 'sidekiq', '> 4', '< 6'
|
34
|
-
spec.add_dependency 'sidekiq-unique-jobs', '~> 5.0'
|
35
|
-
spec.add_dependency 'tabstabs'
|
23
|
+
spec.add_dependency "activerecord", ">= 7.0"
|
24
|
+
spec.add_dependency "activejob", ">= 7.0"
|
25
|
+
spec.add_dependency "aasm"
|
36
26
|
|
37
|
-
spec.add_development_dependency
|
38
|
-
spec.add_development_dependency
|
39
|
-
spec.add_development_dependency
|
40
|
-
spec.add_development_dependency
|
41
|
-
spec.add_development_dependency
|
42
|
-
spec.add_development_dependency
|
43
|
-
spec.add_development_dependency 'mock_redis'
|
27
|
+
spec.add_development_dependency "bundler"
|
28
|
+
spec.add_development_dependency "rake"
|
29
|
+
spec.add_development_dependency "rspec-rails", "~> 7.0"
|
30
|
+
spec.add_development_dependency "sqlite3", "~> 2.1"
|
31
|
+
spec.add_development_dependency "database_cleaner", "~> 2.0"
|
32
|
+
spec.add_development_dependency "standard"
|
44
33
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require_relative "aeternitas_job"
|
2
|
+
|
3
|
+
module Aeternitas
|
4
|
+
# An ActiveJob for cleaning up old metric records.
|
5
|
+
class CleanupOldMetricsJob < AeternitasJob
|
6
|
+
queue_as :maintenance
|
7
|
+
|
8
|
+
def perform
|
9
|
+
Aeternitas::Maintenance.cleanup_old_metrics
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require_relative "aeternitas_job"
|
2
|
+
|
3
|
+
module Aeternitas
|
4
|
+
# An ActiveJob for cleaning up stale lock records.
|
5
|
+
class CleanupStaleLocksJob < AeternitasJob
|
6
|
+
queue_as :maintenance
|
7
|
+
|
8
|
+
def perform
|
9
|
+
Aeternitas::Maintenance.cleanup_stale_locks
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
data/lib/aeternitas/errors.rb
CHANGED
@@ -14,7 +14,6 @@ module Aeternitas
|
|
14
14
|
@original_error = original_error
|
15
15
|
super("#{original_error.class} - #{original_error.message}")
|
16
16
|
end
|
17
|
-
|
18
17
|
end
|
19
18
|
|
20
19
|
# Raised when a source data already exists.
|
@@ -45,4 +44,4 @@ module Aeternitas
|
|
45
44
|
end
|
46
45
|
end
|
47
46
|
end
|
48
|
-
end
|
47
|
+
end
|