pgdice 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -2
- data/.codeclimate.yml +5 -0
- data/.rubocop.yml +6 -1
- data/CHANGELOG.md +14 -0
- data/Guardfile +6 -6
- data/README.md +111 -76
- data/Rakefile +3 -0
- data/examples/config.yml +13 -0
- data/lib/pgdice/approved_tables.rb +63 -0
- data/lib/pgdice/configuration.rb +69 -54
- data/lib/pgdice/configuration_file_loader.rb +62 -0
- data/lib/pgdice/database_connection.rb +17 -9
- data/lib/pgdice/database_connection_factory.rb +20 -0
- data/lib/pgdice/date_helper.rb +39 -0
- data/lib/pgdice/error.rb +71 -0
- data/lib/pgdice/log_helper.rb +37 -0
- data/lib/pgdice/partition_dropper.rb +34 -0
- data/lib/pgdice/partition_dropper_factory.rb +20 -0
- data/lib/pgdice/partition_helper.rb +18 -30
- data/lib/pgdice/partition_helper_factory.rb +24 -0
- data/lib/pgdice/partition_lister.rb +33 -0
- data/lib/pgdice/partition_lister_factory.rb +22 -0
- data/lib/pgdice/partition_manager.rb +62 -58
- data/lib/pgdice/partition_manager_factory.rb +52 -0
- data/lib/pgdice/period_fetcher.rb +31 -0
- data/lib/pgdice/period_fetcher_factory.rb +19 -0
- data/lib/pgdice/pg_slice_manager.rb +44 -29
- data/lib/pgdice/pg_slice_manager_factory.rb +35 -0
- data/lib/pgdice/table.rb +87 -0
- data/lib/pgdice/table_finder.rb +41 -0
- data/lib/pgdice/validation.rb +52 -79
- data/lib/pgdice/validation_factory.rb +21 -0
- data/lib/pgdice/version.rb +1 -1
- data/lib/pgdice.rb +34 -72
- data/pgdice.gemspec +3 -4
- metadata +27 -27
- data/lib/pgdice/table_dropper.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fda4fee4feb3b2d3dd02baa31b524cf14e6047a9c915bc45c9c8abc5f056361a
|
4
|
+
data.tar.gz: 5d12a5c07ebdc686a85d32a567af90c7fa38c98c70cd0ee7986ae70bdaf9a1ce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df344dd28a9a0bf196ced1f01d69e6382164c86f77d53d73886a9243b32177287aea7e0ed6d45a3df34aaea51823654210616947cea4a25369f92db429153cd3
|
7
|
+
data.tar.gz: 86b3055b01c1cbfecd431ed009375650eee9e37d2aea387cc3fad55b47c892a9b72fe1c05d4f29828d62006e640fc65c37afab35e7cab3c91d978ec39aa85961
|
data/.circleci/config.yml
CHANGED
@@ -12,7 +12,7 @@ defaults: &defaults
|
|
12
12
|
DATABASE_USERNAME: pgdice
|
13
13
|
PGDICE_LOG_TARGET: STDOUT
|
14
14
|
|
15
|
-
- image: circleci/postgres:10.
|
15
|
+
- image: circleci/postgres:10.6-alpine-ram
|
16
16
|
environment:
|
17
17
|
POSTGRES_USER: pgdice
|
18
18
|
POSTGRES_DB: pgdice_test
|
@@ -42,7 +42,7 @@ jobs:
|
|
42
42
|
mkdir -p /tmp/test-results
|
43
43
|
TEST_FILES="$(circleci tests glob "test/**/*_test.rb" | circleci tests split --split-by=timings)"
|
44
44
|
|
45
|
-
bundle exec rake test
|
45
|
+
bundle exec rake rubocop test
|
46
46
|
./tmp/cc-test-reporter format-coverage -t simplecov -o tmp/coverage/codeclimate.pgdice.json tmp/coverage/pgdice/.resultset.json
|
47
47
|
|
48
48
|
- store_test_results:
|
data/.codeclimate.yml
ADDED
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# Change Log
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
This project adheres to [Semantic Versioning](http://semver.org/).
|
4
|
+
|
5
|
+
## [v0.2.0] : 2018-09-17
|
6
|
+
### Changelog added
|
7
|
+
|
8
|
+
### Added
|
9
|
+
- Support for overriding configuration parameters. Currently only the `:logger` option is supported.
|
10
|
+
- [DatabaseConnection](lib/pgdice/database_connection.rb) now accepts an opts hash
|
11
|
+
- [PartitionHelper](lib/pgdice/partition_helper.rb) now accepts an opts hash
|
12
|
+
- [PartitionManager](lib/pgdice/partition_manager.rb) now accepts an opts hash
|
13
|
+
- [PgSliceManager](lib/pgdice/pg_slice_manager.rb) now accepts an opts hash
|
14
|
+
- [Validation](lib/pgdice/validation.rb) now accepts an opts hash
|
data/Guardfile
CHANGED
@@ -17,15 +17,15 @@
|
|
17
17
|
#
|
18
18
|
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
|
19
19
|
|
20
|
-
guard :minitest, all_after_pass: true do
|
21
|
-
watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
22
|
-
watch(%r{^test/test_helper\.rb$}) { 'test' }
|
23
|
-
watch(%r{^test/.+_test\.rb$})
|
24
|
-
end
|
25
|
-
|
26
20
|
guard :rubocop, cli: %w[-D -S -a] do
|
27
21
|
watch(/.rubocop.yml/)
|
28
22
|
watch(/.+\.rb$/)
|
29
23
|
watch(/Rakefile/)
|
30
24
|
watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
|
31
25
|
end
|
26
|
+
|
27
|
+
guard :minitest, all_after_pass: true do
|
28
|
+
watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
|
29
|
+
watch(%r{^test/test_helper\.rb$}) { 'test' }
|
30
|
+
watch(%r{^test/.+_test\.rb$})
|
31
|
+
end
|
data/README.md
CHANGED
@@ -2,10 +2,11 @@
|
|
2
2
|
[![Coverage Status](https://coveralls.io/repos/github/IlluminusLimited/pgdice/badge.svg?branch=master)](https://coveralls.io/github/IlluminusLimited/pgdice?branch=master)
|
3
3
|
[![Maintainability](https://api.codeclimate.com/v1/badges/311e005a14749bf2f826/maintainability)](https://codeclimate.com/github/IlluminusLimited/pgdice/maintainability)
|
4
4
|
[![Test Coverage](https://api.codeclimate.com/v1/badges/311e005a14749bf2f826/test_coverage)](https://codeclimate.com/github/IlluminusLimited/pgdice/test_coverage)
|
5
|
+
[![Gem Version](https://badge.fury.io/rb/pgdice.svg)](https://badge.fury.io/rb/pgdice)
|
5
6
|
|
6
7
|
# PgDice
|
7
8
|
|
8
|
-
PgDice is a utility that builds on top of the excellent gem
|
9
|
+
PgDice is a utility for creating and maintaining partitioned database tables that builds on top of the excellent gem
|
9
10
|
[https://github.com/ankane/pgslice](https://github.com/ankane/pgslice)
|
10
11
|
|
11
12
|
PgDice is intended to be used by scheduled background jobs in frameworks like [Sidekiq](https://github.com/mperham/sidekiq)
|
@@ -17,8 +18,8 @@ where logging and clear exception messages are crucial.
|
|
17
18
|
There are some features in this gem which allow you to drop database tables.
|
18
19
|
|
19
20
|
If you choose to use this software without a __tested and working__ backup and restore strategy in place then you
|
20
|
-
are a fool and will pay the price for your negligence.
|
21
|
-
|
21
|
+
are a fool and will pay the price for your negligence. THIS SOFTWARE IS PROVIDED "AS IS",
|
22
|
+
WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. By using this software you agree that the creator,
|
22
23
|
maintainers and any affiliated parties CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
|
23
24
|
|
24
25
|
See the [LICENSE](LICENSE) for more information.
|
@@ -51,58 +52,81 @@ This is an example config from a project using `Sidekiq`
|
|
51
52
|
```ruby
|
52
53
|
require 'pgdice'
|
53
54
|
PgDice.configure do |config|
|
54
|
-
|
55
|
+
# This defaults to STDOUT if you don't specify a logger
|
56
|
+
config.logger_factory = proc { Sidekiq.logger }
|
55
57
|
config.database_url = ENV['PGDICE_DATABASE_URL'] # postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...]
|
56
|
-
|
58
|
+
|
59
|
+
# Set a config file or build the tables manually
|
60
|
+
config.config_file = Rails.root.join('config', 'pgdice.yml') # If you are using rails, else provide the absolute path.
|
61
|
+
# and/or
|
62
|
+
config.approved_tables = PgDice::ApprovedTables.new(
|
63
|
+
PgDice::Table.new(table_name: 'comments', past: 90, future: 7, period: 'day'),
|
64
|
+
PgDice::Table.new(table_name: 'posts', past: 6, future: 2, period: 'month')
|
65
|
+
)
|
57
66
|
end
|
58
67
|
```
|
59
68
|
|
60
69
|
|
61
70
|
#### Configuration Parameters
|
62
71
|
|
63
|
-
`
|
64
|
-
|
65
|
-
`database_url` The postgres database url to connect to. This is required since `pgslice` is used to accomplish some tasks
|
66
|
-
and it only takes a `url` currently.
|
67
|
-
|
68
|
-
`approved_tables` This one is important. If you want to manipulate database tables with this gem you're going to
|
69
|
-
need to add the base table name to this string of comma-separated values.
|
72
|
+
- `database_url` - Required: The postgres database url to connect to.
|
73
|
+
- This is required since `pgslice` requires a postgres `url`.
|
70
74
|
|
71
|
-
`
|
72
|
-
|
73
|
-
dropping tables and adding tables.
|
75
|
+
- `logger_factory` - Optional: A factory that will return a logger to use.
|
76
|
+
- Defaults to `proc { Logger.new(STDOUT) }`
|
74
77
|
|
75
|
-
`
|
76
|
-
|
78
|
+
- `approved_tables` - Optional: (but not really) The tables to allow modification on.
|
79
|
+
- If you want to manipulate database tables with this gem you're going to need to provide this data.
|
80
|
+
- See the [Approved Tables Configuration](#approved-tables-configuration) section for more.
|
77
81
|
|
78
|
-
`
|
82
|
+
- `dry_run` - Optional: Boolean value to control whether changes are executed on the database.
|
83
|
+
- You can set it to either `true` or `false`.
|
84
|
+
- `true` will make PgDice log out the commands but not execute them.
|
79
85
|
|
80
|
-
`
|
86
|
+
- `batch_size` - Optional: Maximum number of tables you can drop in one `drop_old_partitions` call.
|
87
|
+
- Defaults to 7.
|
81
88
|
|
82
89
|
|
83
90
|
#### Advanced Configuration Parameters
|
84
91
|
|
85
|
-
|
86
|
-
|
87
|
-
|
92
|
+
All of the following parameters are optional and honestly you probably will never need to mess with these.
|
93
|
+
|
94
|
+
- `pg_connection` - This is a `PG::Connection` object used for the database queries made from `pgdice`.
|
95
|
+
- By default it will be initialized from the `database_url` if left `nil`.
|
96
|
+
- Keep in mind the dependency `pgslice` will still establish its own connection using the `database_url`
|
97
|
+
so this feature may not be very useful if you are trying to only use one connection for this utility.
|
88
98
|
|
89
|
-
`pg_connection` This is a `PG::Connection` object used for the database queries made from `pgdice`.
|
90
|
-
By default it will be initialized from the `database_url` if left `nil`. Keep in mind the dependency
|
91
|
-
`pgslice` will still establish its own connection using the `database_url` so this feature may not be very
|
92
|
-
useful if you are trying to only use one connection for this utility.
|
93
|
-
|
94
|
-
`database_connection` You can supply your own [DatabaseConnection](lib/pgdice/database_connection.rb) if you like.
|
95
|
-
I'm not sure why you would do this.
|
96
|
-
|
97
|
-
`pg_slice_manager` This is an internal wrapper around `pgslice`. [PgSliceManager](lib/pgdice/pg_slice_manager.rb)
|
98
|
-
This configuration lets you provide your own if you wish. I'm not sure why you would do this.
|
99
|
-
|
100
|
-
`partition_manager` You can supply your own [PartitionManager](lib/pgdice/partition_manager.rb) if you like.
|
101
|
-
I'm not sure why you would do this.
|
102
|
-
|
103
|
-
`partition_helper` You can supply your own [PartitionHelper](lib/pgdice/partition_helper.rb) if you like.
|
104
|
-
I'm not sure why you would do this.
|
105
99
|
|
100
|
+
### Approved Tables Configuration
|
101
|
+
|
102
|
+
In order to maintain the correct number of partitions over time you must configure a
|
103
|
+
[PgDice::Table](lib/pgdice/table.rb).
|
104
|
+
|
105
|
+
An example configuration file has been provided at [config.yml](examples/config.yml) if you would rather
|
106
|
+
declare your `approved_tables` in yaml.
|
107
|
+
|
108
|
+
#### Alternative Approved Tables Configuration
|
109
|
+
|
110
|
+
If you want to declare your [PgDice::ApprovedTables](lib/pgdice/approved_tables.rb) in your configuration
|
111
|
+
block instead, you can build them like so:
|
112
|
+
|
113
|
+
```ruby
|
114
|
+
require 'pgdice'
|
115
|
+
PgDice.configure do |config|
|
116
|
+
config.approved_tables = PgDice::ApprovedTables.new(
|
117
|
+
PgDice::Table.new(table_name: 'comments', # Table name for the (un)partitioned table
|
118
|
+
past: 90, # The minimum number of tables to keep before dropping older tables.
|
119
|
+
future: 7, # Number of future tables to always have.
|
120
|
+
period: 'day', # day, month, year
|
121
|
+
column_name: 'created_at', # Whatever column you'd like to partition on.
|
122
|
+
schema: 'public'), # Schema that this table belongs to.
|
123
|
+
PgDice::Table.new(table_name: 'posts') # Minimum configuration (90 past, 7 future, 'day' period).
|
124
|
+
)
|
125
|
+
end
|
126
|
+
```
|
127
|
+
|
128
|
+
It is possible to use both the configuration block and a file if you so choose.
|
129
|
+
The block will take precedence over the values in the file.
|
106
130
|
|
107
131
|
### Converting existing tables to partitioned tables
|
108
132
|
|
@@ -119,24 +143,28 @@ For more information on what's going on in the background see
|
|
119
143
|
|
120
144
|
|
121
145
|
```ruby
|
122
|
-
PgDice.partition_helper.partition_table
|
123
|
-
past: 30,
|
124
|
-
future: 30,
|
125
|
-
column_name: 'created_at',
|
126
|
-
period: :day)
|
146
|
+
PgDice.partition_helper.partition_table('comments')
|
127
147
|
```
|
128
148
|
|
129
149
|
If you mess up (again you shouldn't use this in production). These two methods are useful for writing tests
|
130
150
|
that work with partitions.
|
131
151
|
|
152
|
+
#### Notes on partition_table
|
153
|
+
|
154
|
+
- You can override values configured in the `PgDice::Table` by passing them in as a hash.
|
155
|
+
- For example if you wanted to create `30` future tables instead of the configured `7` for the `comments` table
|
156
|
+
you could pass in `future: 30`.
|
157
|
+
|
132
158
|
```ruby
|
133
|
-
PgDice.partition_helper.undo_partitioning!(
|
159
|
+
PgDice.partition_helper.undo_partitioning!('comments')
|
134
160
|
```
|
135
161
|
|
136
|
-
|
162
|
+
#### Notes on `partition_table`
|
163
|
+
|
164
|
+
- In `partition_helper` there are versions of the methods that will throw exceptions (ending in `!`) and others
|
137
165
|
that will return a truthy value or `false` if there is a failure.
|
138
166
|
|
139
|
-
`period` can be set to one of these values: `:day`, `:month`, `:year`
|
167
|
+
- `period` can be set to one of these values: `:day`, `:month`, `:year`
|
140
168
|
|
141
169
|
|
142
170
|
### Maintaining partitioned tables
|
@@ -146,30 +174,28 @@ that will return a truthy value or `false` if there is a failure.
|
|
146
174
|
If you have existing tables that need to periodically have more tables added you can run:
|
147
175
|
|
148
176
|
```ruby
|
149
|
-
PgDice.partition_manager.add_new_partitions(
|
177
|
+
PgDice.partition_manager.add_new_partitions('comments')
|
150
178
|
```
|
151
179
|
|
152
|
-
|
180
|
+
##### Notes on `add_new_partitions`
|
181
|
+
|
182
|
+
- The above command would add `7` new tables and their associated indexes all based on the `period` that the
|
153
183
|
partitioned table was defined with.
|
184
|
+
- The example `comments` table we have been using was configured to always keep `7` future partitions above.
|
154
185
|
|
155
186
|
|
156
|
-
#### Listing
|
187
|
+
#### Listing droppable partitions
|
157
188
|
|
158
189
|
Sometimes you just want to know what's out there and if there are tables ready to be dropped.
|
159
190
|
|
160
191
|
To list all eligible tables for dropping you can run:
|
161
192
|
```ruby
|
162
|
-
PgDice.partition_manager.
|
193
|
+
PgDice.partition_manager.list_droppable_partitions('comments')
|
163
194
|
```
|
164
195
|
|
165
|
-
|
166
|
-
```ruby
|
167
|
-
PgDice.partition_manager.list_old_partitions(table_name: 'comments', older_than: 90.days.ago)
|
168
|
-
```
|
196
|
+
##### Notes on `list_droppable_partitions`
|
169
197
|
|
170
|
-
|
171
|
-
It is recommended that you pass it in to be explicit, but you can rely on the configuration
|
172
|
-
mechanism if you so choose.
|
198
|
+
- This method uses the `past` value from the `PgDice::Table` to determine which tables are eligible for dropping.
|
173
199
|
|
174
200
|
|
175
201
|
#### Dropping old tables
|
@@ -179,24 +205,16 @@ _Dropping tables is irreversible! Do this at your own risk!!_
|
|
179
205
|
If you want to drop old tables (after backing them up of course) you can run:
|
180
206
|
|
181
207
|
```ruby
|
182
|
-
PgDice.partition_manager.drop_old_partitions(table_name: 'comments'
|
183
|
-
```
|
184
|
-
|
185
|
-
If you have `active_support` you could do:
|
186
|
-
```ruby
|
187
|
-
PgDice.partition_manager.drop_old_partitions(table_name: 'comments', older_than: 90.days.ago)
|
208
|
+
PgDice.partition_manager.drop_old_partitions(table_name: 'comments')
|
188
209
|
```
|
189
210
|
|
190
|
-
|
191
|
-
|
192
|
-
Technically `older_than` is optional and defaults to `90 days` (see the configuration section).
|
193
|
-
It is recommended that you pass it in to be explicit, but you can rely on the configuration
|
194
|
-
mechanism if you so choose.
|
195
|
-
|
196
|
-
Another good reason to pass in the `older_than` parameter is if you are managing tables that
|
197
|
-
are partiioned by different schemes or have different use-cases
|
198
|
-
e.g. daily vs yearly partitioned tables.
|
211
|
+
##### Notes on `drop_old_partitions`
|
199
212
|
|
213
|
+
- The above example command would drop partitions that exceed the configured `past` table count
|
214
|
+
for the `PgDice::Table`.
|
215
|
+
- The example `comments` table has been configured with `past: 90` tables.
|
216
|
+
So if there were 100 tables older than `today` it would drop up to `batch_size` tables.
|
217
|
+
|
200
218
|
|
201
219
|
#### Validating everything is still working
|
202
220
|
|
@@ -205,7 +223,7 @@ ensure they are actually working correctly.
|
|
205
223
|
|
206
224
|
To validate that your expected number of tables exist, you can run:
|
207
225
|
```ruby
|
208
|
-
PgDice.validation.assert_tables(
|
226
|
+
PgDice.validation.assert_tables('comments', future: 7, past: 90)
|
209
227
|
```
|
210
228
|
|
211
229
|
An [InsufficientTablesError](lib/pgdice.rb) will be raised if any conditions are not met.
|
@@ -215,6 +233,25 @@ still a table from 90 days ago. The above example assumes the table was partitio
|
|
215
233
|
by day.
|
216
234
|
|
217
235
|
|
236
|
+
## FAQ
|
237
|
+
|
238
|
+
1. How do I get a postgres url if I'm running in Rails?
|
239
|
+
```ruby
|
240
|
+
def build_postgres_url
|
241
|
+
config = Rails.configuration.database_configuration
|
242
|
+
host = config[Rails.env]["host"]
|
243
|
+
database = config[Rails.env]["database"]
|
244
|
+
username = config[Rails.env]["username"]
|
245
|
+
password = config[Rails.env]["password"]
|
246
|
+
|
247
|
+
"postgres://#{username}:#{password}@#{host}/#{database}"
|
248
|
+
end
|
249
|
+
```
|
250
|
+
|
251
|
+
1. I'm seeing off-by-one errors for my `validation.assert_tables` calls?
|
252
|
+
- You should make sure your database is configured to use `UTC`.
|
253
|
+
[https://www.postgresql.org/docs/10/datatype-datetime.html](https://www.postgresql.org/docs/10/datatype-datetime.html)
|
254
|
+
|
218
255
|
## Planned Features
|
219
256
|
|
220
257
|
1. Full `PG::Connection` support (no more database URLs).
|
@@ -227,9 +264,7 @@ by day.
|
|
227
264
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests.
|
228
265
|
You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
229
266
|
|
230
|
-
To install this gem onto your local machine, run `bundle exec rake install`.
|
231
|
-
version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
|
232
|
-
push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
267
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
233
268
|
|
234
269
|
|
235
270
|
### Running tests
|
data/Rakefile
CHANGED
data/examples/config.yml
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
approved_tables:
|
2
|
+
- table_name: comments # Table name for the (un)partitioned table
|
3
|
+
past: 1 # The minimum number of tables to keep before dropping older tables.
|
4
|
+
future: 0 # Number of future tables to always have. I like to set this to 7x the period just to be safe.
|
5
|
+
column_name: created_at # Whatever column you'd like to partition on.
|
6
|
+
period: day # day, month, year
|
7
|
+
schema: public
|
8
|
+
- table_name: posts
|
9
|
+
past: 10
|
10
|
+
future: 0
|
11
|
+
column_name: created_at
|
12
|
+
period: day
|
13
|
+
schema: public
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PgDice
|
4
|
+
# Hash-like object to contain approved tables. Adds some convenience validation and a simpleish interface.
|
5
|
+
class ApprovedTables
|
6
|
+
attr_reader :tables
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
def_delegators :@tables, :size, :empty?
|
10
|
+
|
11
|
+
def initialize(*args)
|
12
|
+
@tables = args.flatten.compact
|
13
|
+
|
14
|
+
raise ArgumentError, 'Objects must be a PgDice::Table!' unless tables.all? { |item| item.is_a?(PgDice::Table) }
|
15
|
+
end
|
16
|
+
|
17
|
+
def [](arg)
|
18
|
+
key = check_string_args(arg)
|
19
|
+
tables.select { |table| table.name == key }.first
|
20
|
+
end
|
21
|
+
|
22
|
+
def include?(arg)
|
23
|
+
key = check_string_args(arg)
|
24
|
+
return true if self.[](key)
|
25
|
+
|
26
|
+
false
|
27
|
+
end
|
28
|
+
|
29
|
+
def fetch(arg)
|
30
|
+
key = check_string_args(arg)
|
31
|
+
found_table = self.[](key)
|
32
|
+
raise PgDice::IllegalTableError, "Table name: '#{key}' is not in the list of approved tables!" unless found_table
|
33
|
+
|
34
|
+
found_table
|
35
|
+
end
|
36
|
+
|
37
|
+
def <<(object)
|
38
|
+
raise ArgumentError, 'Objects must be a PgDice::Table!' unless object.is_a?(PgDice::Table)
|
39
|
+
|
40
|
+
object.validate!
|
41
|
+
return self if include?(object.name)
|
42
|
+
|
43
|
+
@tables << object
|
44
|
+
self
|
45
|
+
end
|
46
|
+
|
47
|
+
def smash(table_name, override_parameters)
|
48
|
+
fetch(table_name).smash(override_parameters)
|
49
|
+
end
|
50
|
+
|
51
|
+
def ==(other)
|
52
|
+
tables.sort == other.tables.sort
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def check_string_args(key)
|
58
|
+
raise ArgumentError, 'key must be a String' unless key.is_a?(String)
|
59
|
+
|
60
|
+
key
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/pgdice/configuration.rb
CHANGED
@@ -5,52 +5,52 @@ module PgDice
|
|
5
5
|
class << self
|
6
6
|
attr_accessor :configuration
|
7
7
|
|
8
|
-
def configure
|
8
|
+
def configure(validate_configuration: true)
|
9
9
|
self.configuration ||= PgDice::Configuration.new
|
10
10
|
yield(configuration)
|
11
|
+
configuration.validate! if validate_configuration
|
11
12
|
end
|
12
13
|
end
|
13
14
|
|
14
15
|
# Configuration class which holds all configurable values
|
15
16
|
class Configuration
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
VALUES = { logger: Logger.new(STDOUT),
|
21
|
-
database_url: nil,
|
22
|
-
additional_validators: [],
|
23
|
-
approved_tables: [],
|
24
|
-
older_than: PgDice::Configuration.days_ago(90),
|
25
|
-
dry_run: false,
|
26
|
-
table_drop_batch_size: 7 }.freeze
|
17
|
+
DEFAULT_VALUES ||= { logger_factory: proc { Logger.new(STDOUT) },
|
18
|
+
database_url: nil,
|
19
|
+
dry_run: false,
|
20
|
+
batch_size: 7 }.freeze
|
27
21
|
|
28
22
|
attr_writer :logger,
|
23
|
+
:logger_factory,
|
29
24
|
:database_url,
|
30
|
-
:additional_validators,
|
31
25
|
:approved_tables,
|
32
|
-
:older_than,
|
33
26
|
:dry_run,
|
34
|
-
:
|
35
|
-
:
|
36
|
-
:
|
37
|
-
|
38
|
-
attr_accessor :
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
def initialize(existing_configuration = nil)
|
44
|
-
VALUES.each do |key, value|
|
45
|
-
initialize_value(key, value, existing_configuration)
|
27
|
+
:batch_size,
|
28
|
+
:pg_connection,
|
29
|
+
:config_file_loader
|
30
|
+
|
31
|
+
attr_accessor :config_file
|
32
|
+
|
33
|
+
def initialize(existing_config = nil)
|
34
|
+
DEFAULT_VALUES.each do |key, value|
|
35
|
+
initialize_value(key, value, existing_config)
|
46
36
|
end
|
37
|
+
@approved_tables = PgDice::ApprovedTables.new(existing_config&.approved_tables(eager_load: true)&.tables)
|
47
38
|
initialize_objects
|
48
39
|
end
|
49
40
|
|
50
|
-
def
|
51
|
-
|
41
|
+
def validate!
|
42
|
+
logger_factory
|
43
|
+
database_url
|
44
|
+
database_connection
|
45
|
+
pg_connection
|
46
|
+
batch_size
|
47
|
+
approved_tables
|
48
|
+
end
|
52
49
|
|
53
|
-
|
50
|
+
def logger_factory
|
51
|
+
return @logger_factory if @logger_factory.respond_to?(:call)
|
52
|
+
|
53
|
+
raise PgDice::InvalidConfigurationError, 'logger_factory must be present!'
|
54
54
|
end
|
55
55
|
|
56
56
|
def database_url
|
@@ -59,28 +59,32 @@ module PgDice
|
|
59
59
|
raise PgDice::InvalidConfigurationError, 'database_url must be present!'
|
60
60
|
end
|
61
61
|
|
62
|
-
def
|
63
|
-
return @
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
def approved_tables(eager_load: false)
|
63
|
+
return @approved_tables if eager_load
|
64
|
+
unless @approved_tables.respond_to?(:empty?)
|
65
|
+
raise PgDice::InvalidConfigurationError, 'approved_tables must be an instance of PgDice::ApprovedTables!'
|
66
|
+
end
|
67
67
|
|
68
|
-
|
69
|
-
|
68
|
+
if !config_file_loader.file_loaded? && config_file.present?
|
69
|
+
config_file_loader.load_file
|
70
|
+
@approved_tables
|
71
|
+
end
|
70
72
|
|
71
|
-
|
73
|
+
@approved_tables
|
72
74
|
end
|
73
75
|
|
74
|
-
|
75
|
-
|
76
|
+
# Lazily initialized
|
77
|
+
def pg_connection
|
78
|
+
@pg_connection ||= PG::Connection.new(database_url)
|
79
|
+
return @pg_connection if @pg_connection.respond_to?(:exec)
|
76
80
|
|
77
|
-
raise PgDice::InvalidConfigurationError, '
|
81
|
+
raise PgDice::InvalidConfigurationError, 'pg_connection must be present!'
|
78
82
|
end
|
79
83
|
|
80
|
-
def
|
81
|
-
return @
|
84
|
+
def batch_size
|
85
|
+
return @batch_size.to_i if @batch_size.to_i >= 0
|
82
86
|
|
83
|
-
raise PgDice::InvalidConfigurationError, '
|
87
|
+
raise PgDice::InvalidConfigurationError, 'batch_size must be a non-negative Integer!'
|
84
88
|
end
|
85
89
|
|
86
90
|
def dry_run
|
@@ -89,18 +93,28 @@ module PgDice
|
|
89
93
|
raise PgDice::InvalidConfigurationError, 'dry_run must be either true or false!'
|
90
94
|
end
|
91
95
|
|
92
|
-
def
|
93
|
-
|
96
|
+
def config_file_loader
|
97
|
+
@config_file_loader ||= ConfigurationFileLoader.new(self)
|
98
|
+
end
|
94
99
|
|
95
|
-
|
100
|
+
def logger
|
101
|
+
@logger ||= logger_factory.call
|
96
102
|
end
|
97
103
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
return @pg_connection if @pg_connection.respond_to?(:exec)
|
104
|
+
def partition_manager
|
105
|
+
@partition_manager_factory.call
|
106
|
+
end
|
102
107
|
|
103
|
-
|
108
|
+
def partition_helper
|
109
|
+
@partition_helper_factory.call
|
110
|
+
end
|
111
|
+
|
112
|
+
def validation
|
113
|
+
@validation_factory.call
|
114
|
+
end
|
115
|
+
|
116
|
+
def database_connection
|
117
|
+
@database_connection_factory.call
|
104
118
|
end
|
105
119
|
|
106
120
|
def deep_clone
|
@@ -114,9 +128,10 @@ module PgDice
|
|
114
128
|
end
|
115
129
|
|
116
130
|
def initialize_objects
|
117
|
-
@
|
118
|
-
@
|
119
|
-
@
|
131
|
+
@partition_manager_factory = PgDice::PartitionManagerFactory.new(self)
|
132
|
+
@partition_helper_factory = PgDice::PartitionHelperFactory.new(self)
|
133
|
+
@validation_factory = PgDice::ValidationFactory.new(self)
|
134
|
+
@database_connection_factory = PgDice::DatabaseConnectionFactory.new(self)
|
120
135
|
end
|
121
136
|
end
|
122
137
|
end
|