pgdice 0.4.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +4 -1
- data/.github/ISSUE_TEMPLATE/bug_report.md +27 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.github/workflows/gem-push.yml +20 -0
- data/.github/workflows/ruby.yml +62 -0
- data/.rubocop.yml +12 -2
- data/.ruby-version +1 -1
- data/CHANGELOG.md +21 -0
- data/README.md +22 -26
- data/SECURITY.md +15 -0
- data/examples/aws/README.md +28 -0
- data/examples/aws/cloudformation/scheduled_events.json +59 -0
- data/examples/aws/lib/sqs_listener/default_event_handler.rb +32 -0
- data/examples/aws/lib/sqs_listener/exceptions/unknown_task_error.rb +4 -0
- data/examples/aws/lib/sqs_listener/fallthrough_event_handler.rb +18 -0
- data/examples/aws/lib/sqs_listener/sqs_event_router.rb +32 -0
- data/examples/aws/lib/sqs_listener/typed_event_handler/task_event_handler.rb +46 -0
- data/examples/aws/lib/sqs_listener/typed_event_handler/tasks/database_tasks.rb +37 -0
- data/examples/aws/lib/sqs_listener.rb +47 -0
- data/examples/aws/lib/sqs_message_deleter.rb +32 -0
- data/examples/aws/lib/sqs_poller.rb +67 -0
- data/examples/aws/tasks/poll_sqs.rake +8 -0
- data/examples/aws/workers/pg_dice_worker.rb +54 -0
- data/lib/pgdice/approved_tables.rb +3 -2
- data/lib/pgdice/configuration.rb +5 -5
- data/lib/pgdice/configuration_file_loader.rb +1 -1
- data/lib/pgdice/database_connection_factory.rb +5 -6
- data/lib/pgdice/date_helper.rb +2 -2
- data/lib/pgdice/error.rb +2 -2
- data/lib/pgdice/partition_dropper.rb +1 -1
- data/lib/pgdice/partition_helper.rb +2 -2
- data/lib/pgdice/pg_slice_manager.rb +5 -5
- data/lib/pgdice/query_executor.rb +3 -3
- data/lib/pgdice/query_executor_factory.rb +20 -0
- data/lib/pgdice/table.rb +2 -2
- data/lib/pgdice/version.rb +1 -1
- data/lib/pgdice.rb +1 -0
- data/pgdice.gemspec +26 -22
- metadata +87 -49
- data/.circleci/config.yml +0 -66
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1a9a4ad3435e9d6a9590f0a40a9d9e0329a2d2719ca92b9781f10b301311d31b
|
|
4
|
+
data.tar.gz: 80840f0d48445932a5fd6fe1a908c7e51c5cbbac1d171ea1fdd3c406052b9f63
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e92d88f79eb9630b5e73e89b2b3191ba24699a6845e75e98cc2c25d9027adc4cbeb1981485f229daf5286e6b2e7f0b43b54d4c6c793fe3d33896be19ad1b8416
|
|
7
|
+
data.tar.gz: bfdac98da6f9b8b59f8188bfacb48106683761f651af387497bce7096047022b7e2b621803394779063afaf93626256a3b3fdfacd8fd5aca6d23890b08fe78c0
|
data/.codeclimate.yml
CHANGED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug report
|
|
3
|
+
about: Create a report to help us improve
|
|
4
|
+
title: ''
|
|
5
|
+
labels: needs investigation
|
|
6
|
+
assignees: ''
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
**Describe the bug**
|
|
11
|
+
A clear and concise description of what the bug is.
|
|
12
|
+
|
|
13
|
+
**To Reproduce**
|
|
14
|
+
Steps to reproduce the behavior:
|
|
15
|
+
|
|
16
|
+
**Expected behavior**
|
|
17
|
+
A clear and concise description of what you expected to happen.
|
|
18
|
+
|
|
19
|
+
**Screenshots**
|
|
20
|
+
If applicable, add screenshots to help explain your problem.
|
|
21
|
+
|
|
22
|
+
**Please include this information**
|
|
23
|
+
- Postgres Version: [e.g. 10.6]
|
|
24
|
+
- PgDice Version [e.g. 0.14.3]
|
|
25
|
+
|
|
26
|
+
**Additional context**
|
|
27
|
+
Add any other context about the problem here.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature request
|
|
3
|
+
about: Suggest an idea for this project
|
|
4
|
+
title: ''
|
|
5
|
+
labels: needs investigation
|
|
6
|
+
assignees: ''
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
**Is your feature request related to a problem? Please describe.**
|
|
11
|
+
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
|
12
|
+
|
|
13
|
+
**Describe the solution you'd like**
|
|
14
|
+
A clear and concise description of what you want to happen.
|
|
15
|
+
|
|
16
|
+
**Describe alternatives you've considered**
|
|
17
|
+
A clear and concise description of any alternative solutions or features you've considered.
|
|
18
|
+
|
|
19
|
+
**Additional context**
|
|
20
|
+
Add any other context or screenshots about the feature request here.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: Publish Gem
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v1
|
|
13
|
+
|
|
14
|
+
- name: Release Gem
|
|
15
|
+
if: contains(github.ref, 'refs/tags/v')
|
|
16
|
+
uses: cadwallion/publish-rubygems-action@master
|
|
17
|
+
env:
|
|
18
|
+
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
|
|
19
|
+
RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
|
|
20
|
+
RELEASE_COMMAND: rake release
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
2
|
+
# They are provided by a third-party and are governed by
|
|
3
|
+
# separate terms of service, privacy policy, and support
|
|
4
|
+
# documentation.
|
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
|
7
|
+
|
|
8
|
+
name: Ruby
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [ master ]
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: [ master ]
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
strategy:
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: ['3.0']
|
|
23
|
+
|
|
24
|
+
services:
|
|
25
|
+
# Label used to access the service container
|
|
26
|
+
postgres:
|
|
27
|
+
# Docker Hub image
|
|
28
|
+
image: postgres
|
|
29
|
+
# Provide the password for postgres
|
|
30
|
+
env:
|
|
31
|
+
POSTGRES_USER: pgdice
|
|
32
|
+
POSTGRES_DB: pgdice_test
|
|
33
|
+
POSTGRES_PASSWORD: password
|
|
34
|
+
# Set health checks to wait until postgres has started
|
|
35
|
+
options: >-
|
|
36
|
+
--health-cmd pg_isready
|
|
37
|
+
--health-interval 10s
|
|
38
|
+
--health-timeout 5s
|
|
39
|
+
--health-retries 5
|
|
40
|
+
ports:
|
|
41
|
+
- 5432:5432
|
|
42
|
+
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v2
|
|
45
|
+
- name: Set up Ruby
|
|
46
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
47
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
48
|
+
# uses: ruby/setup-ruby@v1
|
|
49
|
+
uses: ruby/setup-ruby@v1
|
|
50
|
+
with:
|
|
51
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
52
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
53
|
+
# - name: Rubocop # This is blowing up with some stupid panolint gem missing - I have no mention of this anywhere.
|
|
54
|
+
# run: bundle exec rubocop
|
|
55
|
+
|
|
56
|
+
- name: Run tests
|
|
57
|
+
run: bundle exec rake
|
|
58
|
+
env:
|
|
59
|
+
DATABASE_HOST: localhost
|
|
60
|
+
DATABASE_USERNAME: pgdice
|
|
61
|
+
DATABASE_PASSWORD: password
|
|
62
|
+
PGDICE_LOG_TARGET: STDOUT
|
data/.rubocop.yml
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
require:
|
|
2
|
+
- rubocop-performance
|
|
3
|
+
- rubocop-rake
|
|
4
|
+
|
|
5
|
+
Layout/LineLength:
|
|
2
6
|
Max: 120
|
|
3
7
|
|
|
4
8
|
Metrics/BlockLength:
|
|
@@ -8,4 +12,10 @@ Metrics/BlockLength:
|
|
|
8
12
|
# TODO: Evaluate if this is an acceptable parameter list. It's a constructor after all.
|
|
9
13
|
Metrics/ParameterLists:
|
|
10
14
|
Exclude:
|
|
11
|
-
- lib/pgdice/table.rb
|
|
15
|
+
- lib/pgdice/table.rb
|
|
16
|
+
|
|
17
|
+
AllCops:
|
|
18
|
+
NewCops: enable
|
|
19
|
+
SuggestExtensions: false
|
|
20
|
+
Exclude:
|
|
21
|
+
- 'examples/**/*'
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
ruby-
|
|
1
|
+
ruby-3.0.3
|
data/CHANGELOG.md
CHANGED
|
@@ -2,11 +2,32 @@
|
|
|
2
2
|
All notable changes to this project will be documented in this file.
|
|
3
3
|
This project adheres to [Semantic Versioning](http://semver.org/).
|
|
4
4
|
|
|
5
|
+
## [v2.0.0] : 2022-02-09
|
|
6
|
+
### Changes
|
|
7
|
+
- Bump `pg` to `~> 1.3.1`
|
|
8
|
+
- Bump `pgslice` to `0.4.7`
|
|
9
|
+
- Bump various development dependencies
|
|
10
|
+
- Rubocop project
|
|
11
|
+
- Add `required_ruby_version >= 3.0.0`
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
## [v1.0.1] : 2020-01-31
|
|
15
|
+
### Changes
|
|
16
|
+
- Added better examples/docs.
|
|
17
|
+
- Bump `pg` to `~> 1.2.2`
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
## [v0.4.3] : 2019-04-23
|
|
21
|
+
### Changes
|
|
22
|
+
- Fix #31 where the new connection retry code was eagerly initializing the logger
|
|
23
|
+
|
|
24
|
+
|
|
5
25
|
## [v0.4.2] : 2019-04-22
|
|
6
26
|
### Changes
|
|
7
27
|
- Fix #19 where PgDice wouldn't recover from a broken PG::Connection
|
|
8
28
|
by adding new retry behavior
|
|
9
29
|
|
|
30
|
+
|
|
10
31
|
## [v0.4.1] : 2019-03-21
|
|
11
32
|
### Changes
|
|
12
33
|
- Fix bug where partitioning by months would break when the month was < 10
|
data/README.md
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
[](https://coveralls.io/github/IlluminusLimited/pgdice?branch=master)
|
|
3
|
-
[](https://codeclimate.com/github/IlluminusLimited/pgdice/maintainability)
|
|
1
|
+
[](https://github.com/IlluminusLimited/pgdice/actions/workflows/ruby.yml)[](https://codeclimate.com/github/IlluminusLimited/pgdice/maintainability)
|
|
4
2
|
[](https://codeclimate.com/github/IlluminusLimited/pgdice/test_coverage)
|
|
5
3
|
[](https://badge.fury.io/rb/pgdice)
|
|
6
4
|
|
|
@@ -12,18 +10,9 @@ PgDice is a utility for creating and maintaining partitioned database tables tha
|
|
|
12
10
|
PgDice is intended to be used by scheduled background jobs in frameworks like [Sidekiq](https://github.com/mperham/sidekiq)
|
|
13
11
|
where logging and clear exception messages are crucial.
|
|
14
12
|
|
|
13
|
+
# Maintenance status
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
There are some features in this gem which allow you to drop database tables.
|
|
19
|
-
|
|
20
|
-
If you choose to use this software without a __tested and working__ backup and restore strategy in place then you
|
|
21
|
-
are a fool and will pay the price for your negligence. THIS SOFTWARE IS PROVIDED "AS IS",
|
|
22
|
-
WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. By using this software you agree that the creator,
|
|
23
|
-
maintainers and any affiliated parties CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
|
|
24
|
-
|
|
25
|
-
See the [LICENSE](LICENSE) for more information.
|
|
26
|
-
|
|
15
|
+
This project is stable and used daily in production.
|
|
27
16
|
|
|
28
17
|
# Installation
|
|
29
18
|
|
|
@@ -69,7 +58,6 @@ PgDice.configure do |config|
|
|
|
69
58
|
end
|
|
70
59
|
```
|
|
71
60
|
|
|
72
|
-
|
|
73
61
|
### Configuration Parameters
|
|
74
62
|
|
|
75
63
|
- `database_url` - **Required**: The postgres database url to connect to.
|
|
@@ -179,7 +167,6 @@ PgDice.undo_partitioning!('comments')
|
|
|
179
167
|
This method will revert the changes made by partitioning a table. Don't rely on this
|
|
180
168
|
in production if you mess up; you need to test everything thoroughly.
|
|
181
169
|
|
|
182
|
-
|
|
183
170
|
## Maintaining partitioned tables
|
|
184
171
|
|
|
185
172
|
### Adding more tables
|
|
@@ -196,7 +183,6 @@ PgDice.add_new_partitions('comments')
|
|
|
196
183
|
that the partitioned table was defined with.
|
|
197
184
|
- The example `comments` table we have been using was configured to always keep `7` future partitions above.
|
|
198
185
|
|
|
199
|
-
|
|
200
186
|
### Listing droppable partitions
|
|
201
187
|
|
|
202
188
|
Sometimes you just want to know what's out there and if there are tables ready to be dropped.
|
|
@@ -213,7 +199,6 @@ PgDice.list_droppable_partitions_by_batch_size('comments')
|
|
|
213
199
|
|
|
214
200
|
This method will show partitions that are within the configured `batch_size`.
|
|
215
201
|
|
|
216
|
-
|
|
217
202
|
#### Notes on `list_droppable_partitions`
|
|
218
203
|
|
|
219
204
|
- This method uses the `past` value from the `PgDice::Table` to determine which tables are eligible for dropping.
|
|
@@ -235,7 +220,6 @@ PgDice.drop_old_partitions('comments')
|
|
|
235
220
|
for the `PgDice::Table`.
|
|
236
221
|
- The example `comments` table has been configured with `past: 90` tables.
|
|
237
222
|
So if there were 100 tables older than `today` it would drop up to `batch_size` tables.
|
|
238
|
-
|
|
239
223
|
|
|
240
224
|
# Validation
|
|
241
225
|
|
|
@@ -247,7 +231,7 @@ To validate that your expected number of tables exist, you can run:
|
|
|
247
231
|
PgDice.assert_tables('comments')
|
|
248
232
|
```
|
|
249
233
|
|
|
250
|
-
An [InsufficientTablesError](lib/pgdice.rb) will be raised if any conditions are not met.
|
|
234
|
+
An [InsufficientTablesError](lib/pgdice/error.rb) will be raised if any conditions are not met.
|
|
251
235
|
|
|
252
236
|
This will check that there are 7 future tables from now and that there are 90 past tables
|
|
253
237
|
per our configuration above.
|
|
@@ -268,7 +252,6 @@ PgDice.approved_tables
|
|
|
268
252
|
|
|
269
253
|
The [ApprovedTables](lib/pgdice/approved_tables.rb) object responds to the most common enumerable methods.
|
|
270
254
|
|
|
271
|
-
|
|
272
255
|
# Miscellaneous Notes
|
|
273
256
|
|
|
274
257
|
All methods for `PgDice` take a hash which will override whatever values would have been automatically supplied.
|
|
@@ -279,6 +262,12 @@ PgDice.list_droppable_partitions('comments', past: 60)
|
|
|
279
262
|
```
|
|
280
263
|
This example would use `60` instead of the configured value of `90` from the `comments` table we configured above.
|
|
281
264
|
|
|
265
|
+
# Examples
|
|
266
|
+
|
|
267
|
+
1. [Here's an example on how to use PgDice in AWS](examples/aws) and the [README](examples/aws/README.md) which will guide
|
|
268
|
+
you through what is going on.
|
|
269
|
+
|
|
270
|
+
1. [Here's an example on how to write a config.yml for PgDice](examples/config.yml)
|
|
282
271
|
|
|
283
272
|
# FAQ
|
|
284
273
|
|
|
@@ -292,7 +281,7 @@ This example would use `60` instead of the configured value of `90` from the `co
|
|
|
292
281
|
password = config[Rails.env]["password"]
|
|
293
282
|
|
|
294
283
|
"postgres://#{username}:#{password}@#{host}/#{database}"
|
|
295
|
-
end
|
|
284
|
+
end
|
|
296
285
|
```
|
|
297
286
|
|
|
298
287
|
1. I'm seeing off-by-one errors for my `assert_tables` calls?
|
|
@@ -303,7 +292,7 @@ end
|
|
|
303
292
|
|
|
304
293
|
1. Full `PG::Connection` support (no more database URLs).
|
|
305
294
|
1. Non time-range based partitioning. [PgParty](https://github.com/rkrage/pg_party) might be a good option!
|
|
306
|
-
|
|
295
|
+
1. Hourly partitioning
|
|
307
296
|
|
|
308
297
|
# Development
|
|
309
298
|
|
|
@@ -312,7 +301,6 @@ You can also run `bin/console` for an interactive prompt that will allow you to
|
|
|
312
301
|
|
|
313
302
|
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
314
303
|
|
|
315
|
-
|
|
316
304
|
## Running tests
|
|
317
305
|
|
|
318
306
|
You're going to need to have postgres 10 or greater installed.
|
|
@@ -323,7 +311,6 @@ Run the following commands from your terminal. Don't run these on anything but a
|
|
|
323
311
|
1. `createdb pgdice_test`
|
|
324
312
|
1. Now you can run the tests via `guard` or `rake test`
|
|
325
313
|
|
|
326
|
-
|
|
327
314
|
## Contributing
|
|
328
315
|
|
|
329
316
|
Bug reports and pull requests are welcome on GitHub at
|
|
@@ -331,11 +318,20 @@ Bug reports and pull requests are welcome on GitHub at
|
|
|
331
318
|
to be a safe, welcoming space for collaboration, and contributors are expected to adhere to
|
|
332
319
|
the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
|
333
320
|
|
|
334
|
-
|
|
335
321
|
# License
|
|
336
322
|
|
|
337
323
|
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
|
338
324
|
|
|
325
|
+
# Disclaimer
|
|
326
|
+
|
|
327
|
+
There are some features in this gem which allow you to drop database tables, due to the dangerous nature of
|
|
328
|
+
dropping database tables, please ensure you have a tested and working backup and restore strategy.
|
|
329
|
+
|
|
330
|
+
THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED.
|
|
331
|
+
By using this software you agree that the creator, maintainers, and any affiliated parties
|
|
332
|
+
CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
|
|
333
|
+
|
|
334
|
+
See the [LICENSE](LICENSE) for more information.
|
|
339
335
|
|
|
340
336
|
# Code of Conduct
|
|
341
337
|
|
data/SECURITY.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Supported Versions
|
|
4
|
+
|
|
5
|
+
The versions here are currently supported for security patches.
|
|
6
|
+
|
|
7
|
+
| Version | Supported |
|
|
8
|
+
| ------- | ------------------ |
|
|
9
|
+
| latest | :white_check_mark: |
|
|
10
|
+
| < 1.0 | :x: |
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## Reporting a Vulnerability
|
|
14
|
+
|
|
15
|
+
Please open a new issue
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# How can I use PgDice in production?
|
|
2
|
+
|
|
3
|
+
This collection of files is how I use PgDice in production. I'll describe the architecture here so you'll have a place
|
|
4
|
+
to start.
|
|
5
|
+
|
|
6
|
+
1. `tasks/poll_sqs.rake` is run using some sort of process manager like systemd on the ec2 instance. I like to run
|
|
7
|
+
the poll_sqs stuff on my Sidekiq instances because they are the ones who eventually handle the work anyway.
|
|
8
|
+
|
|
9
|
+
1. `lib/sqs_poller.rb` is used to handle the looping logic for the rake task. It invokes `lib/sqs_listener.rb` for each
|
|
10
|
+
iteration.
|
|
11
|
+
|
|
12
|
+
1. `lib/sqs_listener.rb` calls AWS SQS to receive messages and then passes each one into the `lib/sqs_listener/sqs_event_router.rb`
|
|
13
|
+
to be routed to the correct message handler.
|
|
14
|
+
|
|
15
|
+
1. Inside `lib/sqs_listener/sqs_event_router.rb` the message is parsed and passed through a case statement.
|
|
16
|
+
This could be abstracted better but for now if the message has a field of `event_type` and a value of `"task"` then
|
|
17
|
+
the router will send it off to the `TaskEventHandler` which in this case is
|
|
18
|
+
`lib/sqs_listener/typed_event_handler/task_event_handler.rb`
|
|
19
|
+
|
|
20
|
+
1. In the `TaskEventHandler` the task is sent to a handler which responds to the task specified in the message body field `task`.
|
|
21
|
+
|
|
22
|
+
1. The handler for the task (in this case, `DatabaseTasks`) handles the parameters for invoking the Sidekiq worker: `PgDiceWorker`
|
|
23
|
+
|
|
24
|
+
1. Finally, the `PgDiceWorker` is called and handles invoking `PgDice` based on the parameters passed in.
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
Hopefully that wasn't too confusing. There's a lot of steps in here because the system that uses PgDice handles lots
|
|
28
|
+
of different types of SQS events and needs to be as resilient as possible.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"Description": "Deployment stack",
|
|
3
|
+
"Parameters": {
|
|
4
|
+
"PgDiceEnabled": {
|
|
5
|
+
"Type": "String",
|
|
6
|
+
"Description": "The ENABLED/DISABLED state of the cloudwatch scheduled events for PgDice."
|
|
7
|
+
}
|
|
8
|
+
},
|
|
9
|
+
"Resources": {
|
|
10
|
+
"PgDiceDailyAddPartitions": {
|
|
11
|
+
"DependsOn": "IncomingSQS",
|
|
12
|
+
"Type": "AWS::Events::Rule",
|
|
13
|
+
"Properties": {
|
|
14
|
+
"State":{
|
|
15
|
+
"Ref": "PgDiceEnabled"
|
|
16
|
+
},
|
|
17
|
+
"Description": " PgDice daily add partitions",
|
|
18
|
+
"Name": "PgDiceDailyAddPartitions",
|
|
19
|
+
"ScheduleExpression": "rate(1 day)",
|
|
20
|
+
"Targets": [
|
|
21
|
+
{
|
|
22
|
+
"Arn": {
|
|
23
|
+
"Fn::GetAtt": [
|
|
24
|
+
"IncomingSQS",
|
|
25
|
+
"Arn"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
"Id": "PgDiceDailyAddPartitionsId",
|
|
29
|
+
"Input": "{\"event_type\":\"task\",\"task\":\"add_new_partitions\"}"
|
|
30
|
+
}
|
|
31
|
+
]
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"PgDiceDailyDropPartitions": {
|
|
35
|
+
"DependsOn": "IncomingSQS",
|
|
36
|
+
"Type": "AWS::Events::Rule",
|
|
37
|
+
"Properties": {
|
|
38
|
+
"State":{
|
|
39
|
+
"Ref": "PgDiceEnabled"
|
|
40
|
+
},
|
|
41
|
+
"Description": " PgDice daily drop partitions",
|
|
42
|
+
"Name": "PgDiceDailyDropPartitions",
|
|
43
|
+
"ScheduleExpression": "rate(1 day)",
|
|
44
|
+
"Targets": [
|
|
45
|
+
{
|
|
46
|
+
"Arn": {
|
|
47
|
+
"Fn::GetAtt": [
|
|
48
|
+
"IncomingSQS",
|
|
49
|
+
"Arn"
|
|
50
|
+
]
|
|
51
|
+
},
|
|
52
|
+
"Id": "PgDiceDailyDropPartitionsId",
|
|
53
|
+
"Input": "{\"event_type\":\"task\",\"task\":\"drop_old_partitions\"}"
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class DefaultEventHandler
|
|
4
|
+
attr_reader :logger
|
|
5
|
+
|
|
6
|
+
def initialize(opts = {})
|
|
7
|
+
@logger = opts[:logger] ||= Sidekiq.logger
|
|
8
|
+
@fallthrough_event_handler = opts[:fallthrough_event_handler] ||= FallthroughEventHandler.new(logger: logger)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def handle_message(message)
|
|
12
|
+
# Since 'message' is a JSON formatted string, parse the JSON and then get the values under the 'Records' key
|
|
13
|
+
# When JSON parses a string it returns a Ruby Hash (just like a Java HashMap)
|
|
14
|
+
records = JSON.parse(message.body)['Records']
|
|
15
|
+
if records
|
|
16
|
+
process_records(records, message)
|
|
17
|
+
else
|
|
18
|
+
# If the message body doesn't have any entries under the 'Records' key then we don't know what to do.
|
|
19
|
+
@fallthrough_event_handler.call(message)
|
|
20
|
+
end
|
|
21
|
+
rescue StandardError => e
|
|
22
|
+
# If any errors are raised processing this message then call the fallthrough because something went wrong.
|
|
23
|
+
logger.error { "Caught error while handling incoming message. Calling fallthrough_event_handler. Error: #{e}" }
|
|
24
|
+
@fallthrough_event_handler.call(message)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def process_records(records, message)
|
|
30
|
+
# Process default event
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class FallthroughEventHandler
|
|
4
|
+
attr_reader :logger
|
|
5
|
+
|
|
6
|
+
def initialize(opts = {})
|
|
7
|
+
@logger = opts[:logger] ||= Sidekiq.logger
|
|
8
|
+
@sqs_message_deleter = opts[:sqs_message_deleter] ||= SqsMessageDeleter.new(logger: logger)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(message)
|
|
12
|
+
logger.warn do
|
|
13
|
+
"Received sqs message we don't know how to process. Message: #{message}"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
@sqs_message_deleter.call(message.receipt_handle)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Responsible for routing incoming SQS events to the correct handler
|
|
4
|
+
class SqsEventRouter
|
|
5
|
+
attr_reader :logger
|
|
6
|
+
|
|
7
|
+
def initialize(opts = {})
|
|
8
|
+
@logger = opts[:logger] ||= Sidekiq.logger
|
|
9
|
+
@task_event_handler = opts[:task_event_handler] ||= TaskEventHandler.new(logger: logger)
|
|
10
|
+
@default_event_handler = opts[:default_event_handler] ||= DefaultEventHandler.new(logger: logger)
|
|
11
|
+
@sqs_message_deleter = opts[:sqs_message_deleter] ||= SqsMessageDeleter.new(logger: logger)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Handles incoming sqs event, looking for a field of 'event_type'
|
|
15
|
+
# See scheduled_events.json for details on how to create task events from cloudwatch
|
|
16
|
+
def handle_message(message)
|
|
17
|
+
message_body = JSON.parse(message.body).with_indifferent_access
|
|
18
|
+
event_type = message_body[:event_type]
|
|
19
|
+
|
|
20
|
+
logger.tagged(message.receipt_handle) do
|
|
21
|
+
logger.debug { "The received message was: #{message}" }
|
|
22
|
+
|
|
23
|
+
case event_type
|
|
24
|
+
when 'task'
|
|
25
|
+
@task_event_handler.run_task(message_body)
|
|
26
|
+
@sqs_message_deleter.call(message.receipt_handle)
|
|
27
|
+
else
|
|
28
|
+
@default_event_handler.handle_message(message)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class TaskEventHandler
|
|
4
|
+
attr_reader :logger
|
|
5
|
+
|
|
6
|
+
def initialize(opts = {})
|
|
7
|
+
@logger = opts[:logger] ||= Sidekiq.logger
|
|
8
|
+
@task_handlers = [opts[:task_handlers] ||= initialize_default_handlers].flatten.compact
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run_task(message_body_hash)
|
|
12
|
+
task = message_body_hash.fetch(:task).to_sym
|
|
13
|
+
logger.debug { "Running task: #{task}. Searching for task in: #{@task_handlers}" }
|
|
14
|
+
|
|
15
|
+
task_handlers = resolve_task_handlers(task)
|
|
16
|
+
|
|
17
|
+
if task_handlers.blank?
|
|
18
|
+
raise UnknownTaskError, "Could not find task: #{task} in any of the available task_handlers: #{@task_handlers}"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
invoke_task_handler(task_handlers.first, task, message_body_hash.fetch(:parameters, {}))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def resolve_task_handlers(task)
|
|
27
|
+
task_handlers = @task_handlers.select { |task_handler| task_handler.respond_to?(task) }
|
|
28
|
+
|
|
29
|
+
task_handlers.each do |task_handler|
|
|
30
|
+
logger.debug { "Found task handler: #{task_handler.class} that can handle task: #{task}" }
|
|
31
|
+
end
|
|
32
|
+
task_handlers
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def invoke_task_handler(task_handler, task, params)
|
|
36
|
+
logger.debug { "Invoking handler: #{task_handler.class}##{task} with params: #{params}" }
|
|
37
|
+
task_handler.public_send(task, params)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize_default_handlers
|
|
41
|
+
[
|
|
42
|
+
DatabaseTasks.new
|
|
43
|
+
# Other tasks go here
|
|
44
|
+
]
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Tasks that we can use to maintain partitioned tables over time
|
|
4
|
+
# You can override the default params hash by passing it in to the method calls.
|
|
5
|
+
# The default params are defined inside each method.
|
|
6
|
+
#
|
|
7
|
+
# Also, as far as the string keys for hashes go:
|
|
8
|
+
# https://github.com/mperham/sidekiq/wiki/Best-Practices
|
|
9
|
+
# Sidekiq job parameters must be JSON serializable. That means Ruby symbols are
|
|
10
|
+
# lost when they are sent through JSON!
|
|
11
|
+
class DatabaseTasks
|
|
12
|
+
def initialize(opts = {})
|
|
13
|
+
@pgdice = opts[:pgdice] ||= PgDice
|
|
14
|
+
@task_runner = opts[:task_runner] ||= ->(method, params) { PgdiceWorker.perform_async(method, params) }
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def add_new_partitions(params = {})
|
|
18
|
+
all_params = { 'table_names' => table_names, 'only' => 'future', 'validate' => true }.merge(params)
|
|
19
|
+
@task_runner.call('add_new_partitions', all_params)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def drop_old_partitions(params = {})
|
|
23
|
+
all_params = { 'table_names' => table_names, 'only' => 'past', 'validate' => true }.merge(params)
|
|
24
|
+
@task_runner.call('drop_old_partitions', all_params)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def assert_tables(params = {})
|
|
28
|
+
all_params = { 'table_names' => table_names, 'validate' => false }.merge(params)
|
|
29
|
+
@task_runner.call('assert_tables', all_params)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def table_names
|
|
35
|
+
@pgdice.approved_tables.map(&:name)
|
|
36
|
+
end
|
|
37
|
+
end
|