pgdice 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -2
- data/.codeclimate.yml +5 -0
- data/.rubocop.yml +6 -1
- data/CHANGELOG.md +14 -0
- data/Guardfile +6 -6
- data/README.md +111 -76
- data/Rakefile +3 -0
- data/examples/config.yml +13 -0
- data/lib/pgdice/approved_tables.rb +63 -0
- data/lib/pgdice/configuration.rb +69 -54
- data/lib/pgdice/configuration_file_loader.rb +62 -0
- data/lib/pgdice/database_connection.rb +17 -9
- data/lib/pgdice/database_connection_factory.rb +20 -0
- data/lib/pgdice/date_helper.rb +39 -0
- data/lib/pgdice/error.rb +71 -0
- data/lib/pgdice/log_helper.rb +37 -0
- data/lib/pgdice/partition_dropper.rb +34 -0
- data/lib/pgdice/partition_dropper_factory.rb +20 -0
- data/lib/pgdice/partition_helper.rb +18 -30
- data/lib/pgdice/partition_helper_factory.rb +24 -0
- data/lib/pgdice/partition_lister.rb +33 -0
- data/lib/pgdice/partition_lister_factory.rb +22 -0
- data/lib/pgdice/partition_manager.rb +62 -58
- data/lib/pgdice/partition_manager_factory.rb +52 -0
- data/lib/pgdice/period_fetcher.rb +31 -0
- data/lib/pgdice/period_fetcher_factory.rb +19 -0
- data/lib/pgdice/pg_slice_manager.rb +44 -29
- data/lib/pgdice/pg_slice_manager_factory.rb +35 -0
- data/lib/pgdice/table.rb +87 -0
- data/lib/pgdice/table_finder.rb +41 -0
- data/lib/pgdice/validation.rb +52 -79
- data/lib/pgdice/validation_factory.rb +21 -0
- data/lib/pgdice/version.rb +1 -1
- data/lib/pgdice.rb +34 -72
- data/pgdice.gemspec +3 -4
- metadata +27 -27
- data/lib/pgdice/table_dropper.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fda4fee4feb3b2d3dd02baa31b524cf14e6047a9c915bc45c9c8abc5f056361a
|
4
|
+
data.tar.gz: 5d12a5c07ebdc686a85d32a567af90c7fa38c98c70cd0ee7986ae70bdaf9a1ce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df344dd28a9a0bf196ced1f01d69e6382164c86f77d53d73886a9243b32177287aea7e0ed6d45a3df34aaea51823654210616947cea4a25369f92db429153cd3
|
7
|
+
data.tar.gz: 86b3055b01c1cbfecd431ed009375650eee9e37d2aea387cc3fad55b47c892a9b72fe1c05d4f29828d62006e640fc65c37afab35e7cab3c91d978ec39aa85961
|
data/.circleci/config.yml
CHANGED
@@ -12,7 +12,7 @@ defaults: &defaults
|
|
12
12
|
DATABASE_USERNAME: pgdice
|
13
13
|
PGDICE_LOG_TARGET: STDOUT
|
14
14
|
|
15
|
-
- image: circleci/postgres:10.
|
15
|
+
- image: circleci/postgres:10.6-alpine-ram
|
16
16
|
environment:
|
17
17
|
POSTGRES_USER: pgdice
|
18
18
|
POSTGRES_DB: pgdice_test
|
@@ -42,7 +42,7 @@ jobs:
|
|
42
42
|
mkdir -p /tmp/test-results
|
43
43
|
TEST_FILES="$(circleci tests glob "test/**/*_test.rb" | circleci tests split --split-by=timings)"
|
44
44
|
|
45
|
-
bundle exec rake test
|
45
|
+
bundle exec rake rubocop test
|
46
46
|
./tmp/cc-test-reporter format-coverage -t simplecov -o tmp/coverage/codeclimate.pgdice.json tmp/coverage/pgdice/.resultset.json
|
47
47
|
|
48
48
|
- store_test_results:
|
data/.codeclimate.yml
ADDED
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# Change Log
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
This project adheres to [Semantic Versioning](http://semver.org/).
|
4
|
+
|
5
|
+
## [v0.2.0] : 2018-09-17
|
6
|
+
### Changelog added
|
7
|
+
|
8
|
+
### Added
|
9
|
+
- Support for overriding configuration parameters. Currently only the `:logger` option is supported.
|
10
|
+
- [DatabaseConnection](lib/pgdice/database_connection.rb) now accepts an opts hash
|
11
|
+
- [PartitionHelper](lib/pgdice/partition_helper.rb) now accepts an opts hash
|
12
|
+
- [PartitionManager](lib/pgdice/partition_manager.rb) now accepts an opts hash
|
13
|
+
- [PgSliceManager](lib/pgdice/pg_slice_manager.rb) now accepts an opts hash
|
14
|
+
- [Validation](lib/pgdice/validation.rb) now accepts an opts hash
|
data/Guardfile
CHANGED
@@ -17,15 +17,15 @@
|
|
17
17
|
#
|
18
18
|
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
|
19
19
|
|
20
|
-
guard :minitest, all_after_pass: true do
|
21
|
-
watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
22
|
-
watch(%r{^test/test_helper\.rb$}) { 'test' }
|
23
|
-
watch(%r{^test/.+_test\.rb$})
|
24
|
-
end
|
25
|
-
|
26
20
|
guard :rubocop, cli: %w[-D -S -a] do
|
27
21
|
watch(/.rubocop.yml/)
|
28
22
|
watch(/.+\.rb$/)
|
29
23
|
watch(/Rakefile/)
|
30
24
|
watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
|
31
25
|
end
|
26
|
+
|
27
|
+
guard :minitest, all_after_pass: true do
|
28
|
+
watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
29
|
+
watch(%r{^test/test_helper\.rb$}) { 'test' }
|
30
|
+
watch(%r{^test/.+_test\.rb$})
|
31
|
+
end
|
data/README.md
CHANGED
@@ -2,10 +2,11 @@
|
|
2
2
|
[](https://coveralls.io/github/IlluminusLimited/pgdice?branch=master)
|
3
3
|
[](https://codeclimate.com/github/IlluminusLimited/pgdice/maintainability)
|
4
4
|
[](https://codeclimate.com/github/IlluminusLimited/pgdice/test_coverage)
|
5
|
+
[](https://badge.fury.io/rb/pgdice)
|
5
6
|
|
6
7
|
# PgDice
|
7
8
|
|
8
|
-
PgDice is a utility that builds on top of the excellent gem
|
9
|
+
PgDice is a utility for creating and maintaining partitioned database tables that builds on top of the excellent gem
|
9
10
|
[https://github.com/ankane/pgslice](https://github.com/ankane/pgslice)
|
10
11
|
|
11
12
|
PgDice is intended to be used by scheduled background jobs in frameworks like [Sidekiq](https://github.com/mperham/sidekiq)
|
@@ -17,8 +18,8 @@ where logging and clear exception messages are crucial.
|
|
17
18
|
There are some features in this gem which allow you to drop database tables.
|
18
19
|
|
19
20
|
If you choose to use this software without a __tested and working__ backup and restore strategy in place then you
|
20
|
-
are a fool and will pay the price for your negligence.
|
21
|
-
|
21
|
+
are a fool and will pay the price for your negligence. THIS SOFTWARE IS PROVIDED "AS IS",
|
22
|
+
WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. By using this software you agree that the creator,
|
22
23
|
maintainers and any affiliated parties CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
|
23
24
|
|
24
25
|
See the [LICENSE](LICENSE) for more information.
|
@@ -51,58 +52,81 @@ This is an example config from a project using `Sidekiq`
|
|
51
52
|
```ruby
|
52
53
|
require 'pgdice'
|
53
54
|
PgDice.configure do |config|
|
54
|
-
|
55
|
+
# This defaults to STDOUT if you don't specify a logger
|
56
|
+
config.logger_factory = proc { Sidekiq.logger }
|
55
57
|
config.database_url = ENV['PGDICE_DATABASE_URL'] # postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...]
|
56
|
-
|
58
|
+
|
59
|
+
# Set a config file or build the tables manually
|
60
|
+
config.config_file = Rails.root.join('config', 'pgdice.yml') # If you are using rails, else provide the absolute path.
|
61
|
+
# and/or
|
62
|
+
config.approved_tables = PgDice::ApprovedTables.new(
|
63
|
+
PgDice::Table.new(table_name: 'comments', past: 90, future: 7, period: 'day'),
|
64
|
+
PgDice::Table.new(table_name: 'posts', past: 6, future: 2, period: 'month')
|
65
|
+
)
|
57
66
|
end
|
58
67
|
```
|
59
68
|
|
60
69
|
|
61
70
|
#### Configuration Parameters
|
62
71
|
|
63
|
-
`
|
64
|
-
|
65
|
-
`database_url` The postgres database url to connect to. This is required since `pgslice` is used to accomplish some tasks
|
66
|
-
and it only takes a `url` currently.
|
67
|
-
|
68
|
-
`approved_tables` This one is important. If you want to manipulate database tables with this gem you're going to
|
69
|
-
need to add the base table name to this string of comma-separated values.
|
72
|
+
- `database_url` - Required: The postgres database url to connect to.
|
73
|
+
- This is required since `pgslice` requires a postgres `url`.
|
70
74
|
|
71
|
-
`
|
72
|
-
|
73
|
-
dropping tables and adding tables.
|
75
|
+
- `logger_factory` - Optional: A factory that will return a logger to use.
|
76
|
+
- Defaults to `proc { Logger.new(STDOUT) }`
|
74
77
|
|
75
|
-
`
|
76
|
-
|
78
|
+
- `approved_tables` - Optional: (but not really) The tables to allow modification on.
|
79
|
+
- If you want to manipulate database tables with this gem you're going to need to provide this data.
|
80
|
+
- See the [Approved Tables Configuration](#approved-tables-configuration) section for more.
|
77
81
|
|
78
|
-
`
|
82
|
+
- `dry_run` - Optional: Boolean value to control whether changes are executed on the database.
|
83
|
+
- You can set it to either `true` or `false`.
|
84
|
+
- `true` will make PgDice log out the commands but not execute them.
|
79
85
|
|
80
|
-
`
|
86
|
+
- `batch_size` - Optional: Maximum number of tables you can drop in one `drop_old_partitions` call.
|
87
|
+
- Defaults to 7.
|
81
88
|
|
82
89
|
|
83
90
|
#### Advanced Configuration Parameters
|
84
91
|
|
85
|
-
|
86
|
-
|
87
|
-
|
92
|
+
All of the following parameters are optional and honestly you probably will never need to mess with these.
|
93
|
+
|
94
|
+
- `pg_connection` - This is a `PG::Connection` object used for the database queries made from `pgdice`.
|
95
|
+
- By default it will be initialized from the `database_url` if left `nil`.
|
96
|
+
- Keep in mind the dependency `pgslice` will still establish its own connection using the `database_url`
|
97
|
+
so this feature may not be very useful if you are trying to only use one connection for this utility.
|
88
98
|
|
89
|
-
`pg_connection` This is a `PG::Connection` object used for the database queries made from `pgdice`.
|
90
|
-
By default it will be initialized from the `database_url` if left `nil`. Keep in mind the dependency
|
91
|
-
`pgslice` will still establish its own connection using the `database_url` so this feature may not be very
|
92
|
-
useful if you are trying to only use one connection for this utility.
|
93
|
-
|
94
|
-
`database_connection` You can supply your own [DatabaseConnection](lib/pgdice/database_connection.rb) if you like.
|
95
|
-
I'm not sure why you would do this.
|
96
|
-
|
97
|
-
`pg_slice_manager` This is an internal wrapper around `pgslice`. [PgSliceManager](lib/pgdice/pg_slice_manager.rb)
|
98
|
-
This configuration lets you provide your own if you wish. I'm not sure why you would do this.
|
99
|
-
|
100
|
-
`partition_manager` You can supply your own [PartitionManager](lib/pgdice/partition_manager.rb) if you like.
|
101
|
-
I'm not sure why you would do this.
|
102
|
-
|
103
|
-
`partition_helper` You can supply your own [PartitionHelper](lib/pgdice/partition_helper.rb) if you like.
|
104
|
-
I'm not sure why you would do this.
|
105
99
|
|
100
|
+
### Approved Tables Configuration
|
101
|
+
|
102
|
+
In order to maintain the correct number of partitions over time you must configure a
|
103
|
+
[PgDice::Table](lib/pgdice/table.rb).
|
104
|
+
|
105
|
+
An example configuration file has been provided at [config.yml](examples/config.yml) if you would rather
|
106
|
+
declare your `approved_tables` in yaml.
|
107
|
+
|
108
|
+
#### Alternative Approved Tables Configuration
|
109
|
+
|
110
|
+
If you want to declare your [PgDice::ApprovedTables](lib/pgdice/approved_tables.rb) in your configuration
|
111
|
+
block instead, you can build them like so:
|
112
|
+
|
113
|
+
```ruby
|
114
|
+
require 'pgdice'
|
115
|
+
PgDice.configure do |config|
|
116
|
+
config.approved_tables = PgDice::ApprovedTables.new(
|
117
|
+
PgDice::Table.new(table_name: 'comments', # Table name for the (un)partitioned table
|
118
|
+
past: 90, # The minimum number of tables to keep before dropping older tables.
|
119
|
+
future: 7, # Number of future tables to always have.
|
120
|
+
period: 'day', # day, month, year
|
121
|
+
column_name: 'created_at', # Whatever column you'd like to partition on.
|
122
|
+
schema: 'public'), # Schema that this table belongs to.
|
123
|
+
PgDice::Table.new(table_name: 'posts') # Minimum configuration (90 past, 7 future, 'day' period).
|
124
|
+
)
|
125
|
+
end
|
126
|
+
```
|
127
|
+
|
128
|
+
It is possible to use both the configuration block and a file if you so choose.
|
129
|
+
The block will take precedence over the values in the file.
|
106
130
|
|
107
131
|
### Converting existing tables to partitioned tables
|
108
132
|
|
@@ -119,24 +143,28 @@ For more information on what's going on in the background see
|
|
119
143
|
|
120
144
|
|
121
145
|
```ruby
|
122
|
-
PgDice.partition_helper.partition_table
|
123
|
-
past: 30,
|
124
|
-
future: 30,
|
125
|
-
column_name: 'created_at',
|
126
|
-
period: :day)
|
146
|
+
PgDice.partition_helper.partition_table('comments')
|
127
147
|
```
|
128
148
|
|
129
149
|
If you mess up (again you shouldn't use this in production). These two methods are useful for writing tests
|
130
150
|
that work with partitions.
|
131
151
|
|
152
|
+
#### Notes on partition_table
|
153
|
+
|
154
|
+
- You can override values configured in the `PgDice::Table` by passing them in as a hash.
|
155
|
+
- For example if you wanted to create `30` future tables instead of the configured `7` for the `comments` table
|
156
|
+
you could pass in `future: 30`.
|
157
|
+
|
132
158
|
```ruby
|
133
|
-
PgDice.partition_helper.undo_partitioning!(
|
159
|
+
PgDice.partition_helper.undo_partitioning!('comments')
|
134
160
|
```
|
135
161
|
|
136
|
-
|
162
|
+
#### Notes on `partition_table`
|
163
|
+
|
164
|
+
- In `partition_helper` there are versions of the methods that will throw exceptions (ending in `!`) and others
|
137
165
|
that will return a truthy value or `false` if there is a failure.
|
138
166
|
|
139
|
-
`period` can be set to one of these values: `:day`, `:month`, `:year`
|
167
|
+
- `period` can be set to one of these values: `:day`, `:month`, `:year`
|
140
168
|
|
141
169
|
|
142
170
|
### Maintaining partitioned tables
|
@@ -146,30 +174,28 @@ that will return a truthy value or `false` if there is a failure.
|
|
146
174
|
If you have existing tables that need to periodically have more tables added you can run:
|
147
175
|
|
148
176
|
```ruby
|
149
|
-
PgDice.partition_manager.add_new_partitions(
|
177
|
+
PgDice.partition_manager.add_new_partitions('comments')
|
150
178
|
```
|
151
179
|
|
152
|
-
|
180
|
+
##### Notes on `add_new_partitions`
|
181
|
+
|
182
|
+
- The above command would add `7` new tables and their associated indexes all based on the `period` that the
|
153
183
|
partitioned table was defined with.
|
184
|
+
- The example `comments` table we have been using was configured to always keep `7` future partitions above.
|
154
185
|
|
155
186
|
|
156
|
-
#### Listing
|
187
|
+
#### Listing droppable partitions
|
157
188
|
|
158
189
|
Sometimes you just want to know what's out there and if there are tables ready to be dropped.
|
159
190
|
|
160
191
|
To list all eligible tables for dropping you can run:
|
161
192
|
```ruby
|
162
|
-
PgDice.partition_manager.
|
193
|
+
PgDice.partition_manager.list_droppable_partitions('comments')
|
163
194
|
```
|
164
195
|
|
165
|
-
|
166
|
-
```ruby
|
167
|
-
PgDice.partition_manager.list_old_partitions(table_name: 'comments', older_than: 90.days.ago)
|
168
|
-
```
|
196
|
+
##### Notes on `list_droppable_partitions`
|
169
197
|
|
170
|
-
|
171
|
-
It is recommended that you pass it in to be explicit, but you can rely on the configuration
|
172
|
-
mechanism if you so choose.
|
198
|
+
- This method uses the `past` value from the `PgDice::Table` to determine which tables are eligible for dropping.
|
173
199
|
|
174
200
|
|
175
201
|
#### Dropping old tables
|
@@ -179,24 +205,16 @@ _Dropping tables is irreversible! Do this at your own risk!!_
|
|
179
205
|
If you want to drop old tables (after backing them up of course) you can run:
|
180
206
|
|
181
207
|
```ruby
|
182
|
-
PgDice.partition_manager.drop_old_partitions(table_name: 'comments'
|
183
|
-
```
|
184
|
-
|
185
|
-
If you have `active_support` you could do:
|
186
|
-
```ruby
|
187
|
-
PgDice.partition_manager.drop_old_partitions(table_name: 'comments', older_than: 90.days.ago)
|
208
|
+
PgDice.partition_manager.drop_old_partitions(table_name: 'comments')
|
188
209
|
```
|
189
210
|
|
190
|
-
|
191
|
-
|
192
|
-
Technically `older_than` is optional and defaults to `90 days` (see the configuration section).
|
193
|
-
It is recommended that you pass it in to be explicit, but you can rely on the configuration
|
194
|
-
mechanism if you so choose.
|
195
|
-
|
196
|
-
Another good reason to pass in the `older_than` parameter is if you are managing tables that
|
197
|
-
are partiioned by different schemes or have different use-cases
|
198
|
-
e.g. daily vs yearly partitioned tables.
|
211
|
+
##### Notes on `drop_old_partitions`
|
199
212
|
|
213
|
+
- The above example command would drop partitions that exceed the configured `past` table count
|
214
|
+
for the `PgDice::Table`.
|
215
|
+
- The example `comments` table has been configured with `past: 90` tables.
|
216
|
+
So if there were 100 tables older than `today` it would drop up to `batch_size` tables.
|
217
|
+
|
200
218
|
|
201
219
|
#### Validating everything is still working
|
202
220
|
|
@@ -205,7 +223,7 @@ ensure they are actually working correctly.
|
|
205
223
|
|
206
224
|
To validate that your expected number of tables exist, you can run:
|
207
225
|
```ruby
|
208
|
-
PgDice.validation.assert_tables(
|
226
|
+
PgDice.validation.assert_tables('comments', future: 7, past: 90)
|
209
227
|
```
|
210
228
|
|
211
229
|
An [InsufficientTablesError](lib/pgdice.rb) will be raised if any conditions are not met.
|
@@ -215,6 +233,25 @@ still a table from 90 days ago. The above example assumes the table was partitio
|
|
215
233
|
by day.
|
216
234
|
|
217
235
|
|
236
|
+
## FAQ
|
237
|
+
|
238
|
+
1. How do I get a postgres url if I'm running in Rails?
|
239
|
+
```ruby
|
240
|
+
def build_postgres_url
|
241
|
+
config = Rails.configuration.database_configuration
|
242
|
+
host = config[Rails.env]["host"]
|
243
|
+
database = config[Rails.env]["database"]
|
244
|
+
username = config[Rails.env]["username"]
|
245
|
+
password = config[Rails.env]["password"]
|
246
|
+
|
247
|
+
"postgres://#{username}:#{password}@#{host}/#{database}"
|
248
|
+
end
|
249
|
+
```
|
250
|
+
|
251
|
+
1. I'm seeing off-by-one errors for my `validation.assert_tables` calls?
|
252
|
+
- You should make sure your database is configured to use `UTC`.
|
253
|
+
[https://www.postgresql.org/docs/10/datatype-datetime.html](https://www.postgresql.org/docs/10/datatype-datetime.html)
|
254
|
+
|
218
255
|
## Planned Features
|
219
256
|
|
220
257
|
1. Full `PG::Connection` support (no more database URLs).
|
@@ -227,9 +264,7 @@ by day.
|
|
227
264
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests.
|
228
265
|
You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
229
266
|
|
230
|
-
To install this gem onto your local machine, run `bundle exec rake install`.
|
231
|
-
version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
|
232
|
-
push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
267
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
233
268
|
|
234
269
|
|
235
270
|
### Running tests
|
data/Rakefile
CHANGED
data/examples/config.yml
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
approved_tables:
|
2
|
+
- table_name: comments # Table name for the (un)partitioned table
|
3
|
+
past: 1 # The minimum number of tables to keep before dropping older tables.
|
4
|
+
future: 0 # Number of future tables to always have. I like to set this to 7x the period just to be safe.
|
5
|
+
column_name: created_at # Whatever column you'd like to partition on.
|
6
|
+
period: day # day, month, year
|
7
|
+
schema: public
|
8
|
+
- table_name: posts
|
9
|
+
past: 10
|
10
|
+
future: 0
|
11
|
+
column_name: created_at
|
12
|
+
period: day
|
13
|
+
schema: public
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PgDice
|
4
|
+
# Hash-like object to contain approved tables. Adds some convenience validation and a simpleish interface.
|
5
|
+
class ApprovedTables
|
6
|
+
attr_reader :tables
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
def_delegators :@tables, :size, :empty?
|
10
|
+
|
11
|
+
def initialize(*args)
|
12
|
+
@tables = args.flatten.compact
|
13
|
+
|
14
|
+
raise ArgumentError, 'Objects must be a PgDice::Table!' unless tables.all? { |item| item.is_a?(PgDice::Table) }
|
15
|
+
end
|
16
|
+
|
17
|
+
def [](arg)
|
18
|
+
key = check_string_args(arg)
|
19
|
+
tables.select { |table| table.name == key }.first
|
20
|
+
end
|
21
|
+
|
22
|
+
def include?(arg)
|
23
|
+
key = check_string_args(arg)
|
24
|
+
return true if self.[](key)
|
25
|
+
|
26
|
+
false
|
27
|
+
end
|
28
|
+
|
29
|
+
def fetch(arg)
|
30
|
+
key = check_string_args(arg)
|
31
|
+
found_table = self.[](key)
|
32
|
+
raise PgDice::IllegalTableError, "Table name: '#{key}' is not in the list of approved tables!" unless found_table
|
33
|
+
|
34
|
+
found_table
|
35
|
+
end
|
36
|
+
|
37
|
+
def <<(object)
|
38
|
+
raise ArgumentError, 'Objects must be a PgDice::Table!' unless object.is_a?(PgDice::Table)
|
39
|
+
|
40
|
+
object.validate!
|
41
|
+
return self if include?(object.name)
|
42
|
+
|
43
|
+
@tables << object
|
44
|
+
self
|
45
|
+
end
|
46
|
+
|
47
|
+
def smash(table_name, override_parameters)
|
48
|
+
fetch(table_name).smash(override_parameters)
|
49
|
+
end
|
50
|
+
|
51
|
+
def ==(other)
|
52
|
+
tables.sort == other.tables.sort
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def check_string_args(key)
|
58
|
+
raise ArgumentError, 'key must be a String' unless key.is_a?(String)
|
59
|
+
|
60
|
+
key
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/pgdice/configuration.rb
CHANGED
@@ -5,52 +5,52 @@ module PgDice
|
|
5
5
|
class << self
|
6
6
|
attr_accessor :configuration
|
7
7
|
|
8
|
-
def configure
|
8
|
+
def configure(validate_configuration: true)
|
9
9
|
self.configuration ||= PgDice::Configuration.new
|
10
10
|
yield(configuration)
|
11
|
+
configuration.validate! if validate_configuration
|
11
12
|
end
|
12
13
|
end
|
13
14
|
|
14
15
|
# Configuration class which holds all configurable values
|
15
16
|
class Configuration
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
VALUES = { logger: Logger.new(STDOUT),
|
21
|
-
database_url: nil,
|
22
|
-
additional_validators: [],
|
23
|
-
approved_tables: [],
|
24
|
-
older_than: PgDice::Configuration.days_ago(90),
|
25
|
-
dry_run: false,
|
26
|
-
table_drop_batch_size: 7 }.freeze
|
17
|
+
DEFAULT_VALUES ||= { logger_factory: proc { Logger.new(STDOUT) },
|
18
|
+
database_url: nil,
|
19
|
+
dry_run: false,
|
20
|
+
batch_size: 7 }.freeze
|
27
21
|
|
28
22
|
attr_writer :logger,
|
23
|
+
:logger_factory,
|
29
24
|
:database_url,
|
30
|
-
:additional_validators,
|
31
25
|
:approved_tables,
|
32
|
-
:older_than,
|
33
26
|
:dry_run,
|
34
|
-
:
|
35
|
-
:
|
36
|
-
:
|
37
|
-
|
38
|
-
attr_accessor :
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
def initialize(existing_configuration = nil)
|
44
|
-
VALUES.each do |key, value|
|
45
|
-
initialize_value(key, value, existing_configuration)
|
27
|
+
:batch_size,
|
28
|
+
:pg_connection,
|
29
|
+
:config_file_loader
|
30
|
+
|
31
|
+
attr_accessor :config_file
|
32
|
+
|
33
|
+
def initialize(existing_config = nil)
|
34
|
+
DEFAULT_VALUES.each do |key, value|
|
35
|
+
initialize_value(key, value, existing_config)
|
46
36
|
end
|
37
|
+
@approved_tables = PgDice::ApprovedTables.new(existing_config&.approved_tables(eager_load: true)&.tables)
|
47
38
|
initialize_objects
|
48
39
|
end
|
49
40
|
|
50
|
-
def
|
51
|
-
|
41
|
+
def validate!
|
42
|
+
logger_factory
|
43
|
+
database_url
|
44
|
+
database_connection
|
45
|
+
pg_connection
|
46
|
+
batch_size
|
47
|
+
approved_tables
|
48
|
+
end
|
52
49
|
|
53
|
-
|
50
|
+
def logger_factory
|
51
|
+
return @logger_factory if @logger_factory.respond_to?(:call)
|
52
|
+
|
53
|
+
raise PgDice::InvalidConfigurationError, 'logger_factory must be present!'
|
54
54
|
end
|
55
55
|
|
56
56
|
def database_url
|
@@ -59,28 +59,32 @@ module PgDice
|
|
59
59
|
raise PgDice::InvalidConfigurationError, 'database_url must be present!'
|
60
60
|
end
|
61
61
|
|
62
|
-
def
|
63
|
-
return @
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
def approved_tables(eager_load: false)
|
63
|
+
return @approved_tables if eager_load
|
64
|
+
unless @approved_tables.respond_to?(:empty?)
|
65
|
+
raise PgDice::InvalidConfigurationError, 'approved_tables must be an instance of PgDice::ApprovedTables!'
|
66
|
+
end
|
67
67
|
|
68
|
-
|
69
|
-
|
68
|
+
if !config_file_loader.file_loaded? && config_file.present?
|
69
|
+
config_file_loader.load_file
|
70
|
+
@approved_tables
|
71
|
+
end
|
70
72
|
|
71
|
-
|
73
|
+
@approved_tables
|
72
74
|
end
|
73
75
|
|
74
|
-
|
75
|
-
|
76
|
+
# Lazily initialized
|
77
|
+
def pg_connection
|
78
|
+
@pg_connection ||= PG::Connection.new(database_url)
|
79
|
+
return @pg_connection if @pg_connection.respond_to?(:exec)
|
76
80
|
|
77
|
-
raise PgDice::InvalidConfigurationError, '
|
81
|
+
raise PgDice::InvalidConfigurationError, 'pg_connection must be present!'
|
78
82
|
end
|
79
83
|
|
80
|
-
def
|
81
|
-
return @
|
84
|
+
def batch_size
|
85
|
+
return @batch_size.to_i if @batch_size.to_i >= 0
|
82
86
|
|
83
|
-
raise PgDice::InvalidConfigurationError, '
|
87
|
+
raise PgDice::InvalidConfigurationError, 'batch_size must be a non-negative Integer!'
|
84
88
|
end
|
85
89
|
|
86
90
|
def dry_run
|
@@ -89,18 +93,28 @@ module PgDice
|
|
89
93
|
raise PgDice::InvalidConfigurationError, 'dry_run must be either true or false!'
|
90
94
|
end
|
91
95
|
|
92
|
-
def
|
93
|
-
|
96
|
+
def config_file_loader
|
97
|
+
@config_file_loader ||= ConfigurationFileLoader.new(self)
|
98
|
+
end
|
94
99
|
|
95
|
-
|
100
|
+
def logger
|
101
|
+
@logger ||= logger_factory.call
|
96
102
|
end
|
97
103
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
return @pg_connection if @pg_connection.respond_to?(:exec)
|
104
|
+
def partition_manager
|
105
|
+
@partition_manager_factory.call
|
106
|
+
end
|
102
107
|
|
103
|
-
|
108
|
+
def partition_helper
|
109
|
+
@partition_helper_factory.call
|
110
|
+
end
|
111
|
+
|
112
|
+
def validation
|
113
|
+
@validation_factory.call
|
114
|
+
end
|
115
|
+
|
116
|
+
def database_connection
|
117
|
+
@database_connection_factory.call
|
104
118
|
end
|
105
119
|
|
106
120
|
def deep_clone
|
@@ -114,9 +128,10 @@ module PgDice
|
|
114
128
|
end
|
115
129
|
|
116
130
|
def initialize_objects
|
117
|
-
@
|
118
|
-
@
|
119
|
-
@
|
131
|
+
@partition_manager_factory = PgDice::PartitionManagerFactory.new(self)
|
132
|
+
@partition_helper_factory = PgDice::PartitionHelperFactory.new(self)
|
133
|
+
@validation_factory = PgDice::ValidationFactory.new(self)
|
134
|
+
@database_connection_factory = PgDice::DatabaseConnectionFactory.new(self)
|
120
135
|
end
|
121
136
|
end
|
122
137
|
end
|