tablature 0.1.1 → 1.0.0.pre2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +68 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +1 -1
- data/.yardopts +1 -0
- data/Gemfile +8 -0
- data/README.md +99 -2
- data/lib/tablature.rb +8 -1
- data/lib/tablature/adapters/postgres.rb +76 -5
- data/lib/tablature/adapters/postgres/connection.rb +14 -0
- data/lib/tablature/adapters/postgres/errors.rb +19 -2
- data/lib/tablature/adapters/postgres/handlers/base.rb +4 -0
- data/lib/tablature/adapters/postgres/handlers/list.rb +57 -3
- data/lib/tablature/adapters/postgres/handlers/range.rb +62 -3
- data/lib/tablature/adapters/postgres/indexes.rb +96 -0
- data/lib/tablature/adapters/postgres/partitioned_tables.rb +20 -8
- data/lib/tablature/adapters/postgres/quoting.rb +5 -0
- data/lib/tablature/command_recorder.rb +93 -0
- data/lib/tablature/model.rb +70 -0
- data/lib/tablature/partition.rb +23 -0
- data/lib/tablature/partitioned_table.rb +51 -7
- data/lib/tablature/schema_dumper.rb +66 -3
- data/lib/tablature/statements.rb +67 -11
- data/lib/tablature/version.rb +1 -1
- data/tablature.gemspec +6 -4
- metadata +18 -19
- data/.circleci/config.yml +0 -81
- data/Gemfile.lock +0 -106
- data/bin/console +0 -10
- data/bin/setup +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a72b6d120e1af25bf4b044347d02c552898ede4de5082a01f8b4ec3e24f4f6b0
|
4
|
+
data.tar.gz: e011684eadd4c9f39b83db417c6ef282f5291ed947c23651e8e5c70fcb76aa53
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9b49e89742a179e6f50ccf28789bdb66999f5edb56a4cdcf57c273abc055b5bdc9bb6cbbb754639831613d0b90c034d9ee16c07ea6f56e14ca276a819081296f
|
7
|
+
data.tar.gz: 540d21c49ac1f7e74da0abb9c697cfbc94c72452408acfb891edaf377a5b199bda57ae670dfd051cf266e8c4da86ea51e699a5e4bc25b6e3f0cfc9ade27f378c
|
@@ -0,0 +1,68 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: master
|
6
|
+
pull_request:
|
7
|
+
branches: "*"
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
build:
|
11
|
+
name: Ruby ${{ matrix.ruby }}, Rails ${{ matrix.rails }}, Postgres ${{ matrix.postgres }}
|
12
|
+
|
13
|
+
strategy:
|
14
|
+
fail-fast: false
|
15
|
+
matrix:
|
16
|
+
ruby: ["2.5", "2.7"]
|
17
|
+
rails: ["5.2", "6.0", "master"]
|
18
|
+
postgres: ["10.12", "11.7", "12.2"]
|
19
|
+
include:
|
20
|
+
- postgres: "10.12"
|
21
|
+
rspec_tag: --tag ~postgres_11
|
22
|
+
- rails: "master"
|
23
|
+
continue-on-error: true
|
24
|
+
|
25
|
+
runs-on: ubuntu-latest
|
26
|
+
|
27
|
+
services:
|
28
|
+
postgres:
|
29
|
+
image: postgres:${{ matrix.postgres }}-alpine
|
30
|
+
env:
|
31
|
+
POSTGRES_PASSWORD: postgres
|
32
|
+
ports:
|
33
|
+
- 5432:5432
|
34
|
+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
35
|
+
|
36
|
+
env:
|
37
|
+
RAILS_VERSION: ${{ matrix.rails }}
|
38
|
+
POSTGRES_USER: "postgres"
|
39
|
+
POSTGRES_PASSWORD: "postgres"
|
40
|
+
CI: "true"
|
41
|
+
|
42
|
+
steps:
|
43
|
+
- name: Checkout
|
44
|
+
uses: actions/checkout@v2
|
45
|
+
|
46
|
+
- name: Install dependent libraries
|
47
|
+
run: sudo apt-get install libpq-dev
|
48
|
+
|
49
|
+
- name: Install Ruby ${{ matrix.ruby }}
|
50
|
+
uses: ruby/setup-ruby@v1.31.0
|
51
|
+
with:
|
52
|
+
ruby-version: ${{ matrix.ruby }}
|
53
|
+
|
54
|
+
- name: Generate lockfile
|
55
|
+
run: bundle lock
|
56
|
+
|
57
|
+
- name: Cache dependencies
|
58
|
+
uses: actions/cache@v1
|
59
|
+
with:
|
60
|
+
path: vendor/bundle
|
61
|
+
key: bundle-${{ hashFiles('Gemfile.lock') }}
|
62
|
+
|
63
|
+
- name: Set up Tablature
|
64
|
+
run: bin/setup
|
65
|
+
|
66
|
+
- name: Run tests
|
67
|
+
run: bundle exec rspec ${{ matrix.rspec_tag }}
|
68
|
+
continue-on-error: ${{ matrix.continue-on-error }}
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/.yardopts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
-m markdown
|
data/Gemfile
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
source 'https://rubygems.org'
|
1
2
|
gemspec
|
2
3
|
|
3
4
|
gem 'pry'
|
5
|
+
gem 'rubocop'
|
6
|
+
|
7
|
+
rails_version = ENV.fetch('RAILS_VERSION', '6.0')
|
8
|
+
|
9
|
+
rails_constraint = rails_version == 'master' ? { github: 'rails/rails' } : "~> #{rails_version}.0"
|
10
|
+
|
11
|
+
gem 'rails', rails_constraint
|
data/README.md
CHANGED
@@ -7,7 +7,7 @@ It ships with Postgres support and can easily supports other databases through a
|
|
7
7
|
|
8
8
|
##### Requirements
|
9
9
|
|
10
|
-
Tablature requires Rails 5+ and Postgres 10+.
|
10
|
+
Tablature requires Ruby 2.5+, Rails 5+ and Postgres 10+.
|
11
11
|
|
12
12
|
##### Installation
|
13
13
|
|
@@ -55,10 +55,107 @@ class CreateEvents < ActiveRecord::Migration[5.0]
|
|
55
55
|
# Create partitions with the bounds of the partition.
|
56
56
|
create_list_partition_of :events_by_list,
|
57
57
|
name: 'events_list_y2018m12', values: (Date.parse('2018-12-01')..Date.parse('2018-12-31')).to_a
|
58
|
+
|
58
59
|
end
|
59
60
|
|
60
61
|
def down
|
61
|
-
drop_table :
|
62
|
+
drop_table :events_by_range
|
63
|
+
drop_table :events_by_list
|
64
|
+
end
|
65
|
+
end
|
66
|
+
```
|
67
|
+
|
68
|
+
### Having a partition back a model
|
69
|
+
|
70
|
+
In your migration:
|
71
|
+
```ruby
|
72
|
+
# db/migrate/create_events.rb
|
73
|
+
class CreateEvents < ActiveRecord::Migration
|
74
|
+
def change
|
75
|
+
# You can use blocks when the partition key are SQL expression instead of
|
76
|
+
# being only a field.
|
77
|
+
create_range_partition :events, partition_key: -> { '(timestamp::DATE)' } do |t|
|
78
|
+
t.string :event_type, null: false
|
79
|
+
t.integer :value, null: false
|
80
|
+
t.datetime :timestamp, null: false
|
81
|
+
t.timestamps
|
82
|
+
end
|
83
|
+
|
84
|
+
create_range_partition_of :events,
|
85
|
+
name: 'events_y2018m12', range_start: '2018-12-01', range_end: '2019-01-01'
|
86
|
+
|
87
|
+
create_range_partition_of :events,
|
88
|
+
name: 'events_y2019m01', range_start: '2019-01-01', range_end: '2019-02-01'
|
89
|
+
end
|
90
|
+
end
|
91
|
+
```
|
92
|
+
|
93
|
+
In your model, calling one of `range_partition` or `list_partition` to inject
|
94
|
+
methods:
|
95
|
+
```ruby
|
96
|
+
# app/models/event.rb
|
97
|
+
class Event < ApplicationRecord
|
98
|
+
range_partition
|
99
|
+
end
|
100
|
+
```
|
101
|
+
|
102
|
+
Finally, you can now list the partitions :
|
103
|
+
```ruby
|
104
|
+
>> Event.partitions
|
105
|
+
# => ["events_y2018m12", "events_y2019m01"]
|
106
|
+
```
|
107
|
+
|
108
|
+
You can also create new partitions directly from the model :
|
109
|
+
```ruby
|
110
|
+
>> Event.create_range_partition(
|
111
|
+
name: 'events_y2019m02',
|
112
|
+
range_start: '2019-02-01'.to_date,
|
113
|
+
range_end: '2019-03-01'.to_date
|
114
|
+
)
|
115
|
+
# => ...
|
116
|
+
>> Event.partitions
|
117
|
+
# => ["events_y2018m12", "events_y2019m01", "events_y2019m02"]
|
118
|
+
```
|
119
|
+
|
120
|
+
### Partitioning an existing table
|
121
|
+
Start by renaming your table and create the partition table:
|
122
|
+
```ruby
|
123
|
+
class PartitionEvents < ActiveRecord::Migration
|
124
|
+
def change
|
125
|
+
# Get the bounds of the events.
|
126
|
+
min_month = Event.minimum(:timestamp).beginning_of_month.to_date
|
127
|
+
max_month = Event.maximum(:timestamp).beginning_of_month.to_date
|
128
|
+
|
129
|
+
# Create the partition bounds based on the existing data. In this example,
|
130
|
+
# we generate an array with the ranges.
|
131
|
+
months = min_month.upto(max_month).uniq(&:beginning_of_month)
|
132
|
+
|
133
|
+
# Rename the existing table.
|
134
|
+
rename_table :events, :old_events
|
135
|
+
|
136
|
+
# Create the partitioned table.
|
137
|
+
create_range_partition :events, partition_key: -> { '(timestamp::DATE)' } do |t|
|
138
|
+
t.string :event_type, null: false
|
139
|
+
t.integer :value, null: false
|
140
|
+
t.datetime :timestamp, null: false
|
141
|
+
t.timestamps
|
142
|
+
end
|
143
|
+
|
144
|
+
# Create the partitions based on the bounds generated before:
|
145
|
+
months.each do |month|
|
146
|
+
# Creates a name like "events_y2018m12"
|
147
|
+
partition_name = "events_y#{month.year}m#{month.month}"
|
148
|
+
|
149
|
+
create_range_partition_of :events,
|
150
|
+
name: partition_name, range_start: month, range_end: month.next_month
|
151
|
+
end
|
152
|
+
|
153
|
+
# Finally, add the rows from the old table to the new partitioned table.
|
154
|
+
# This might take some time depending on the size of your old table.
|
155
|
+
execute(<<~SQL)
|
156
|
+
INSERT INTO events
|
157
|
+
SELECT * FROM old_events
|
158
|
+
SQL
|
62
159
|
end
|
63
160
|
end
|
64
161
|
```
|
data/lib/tablature.rb
CHANGED
@@ -2,6 +2,7 @@ require 'tablature/adapters/postgres'
|
|
2
2
|
require 'tablature/command_recorder'
|
3
3
|
require 'tablature/configuration'
|
4
4
|
require 'tablature/model'
|
5
|
+
require 'tablature/partition'
|
5
6
|
require 'tablature/partitioned_table'
|
6
7
|
require 'tablature/railtie'
|
7
8
|
require 'tablature/schema_dumper'
|
@@ -20,7 +21,7 @@ module Tablature
|
|
20
21
|
ActiveRecord::ConnectionAdapters::AbstractAdapter.include Tablature::Statements
|
21
22
|
ActiveRecord::Migration::CommandRecorder.include Tablature::CommandRecorder
|
22
23
|
ActiveRecord::SchemaDumper.prepend Tablature::SchemaDumper
|
23
|
-
ActiveRecord::Base.
|
24
|
+
ActiveRecord::Base.include Tablature::Model
|
24
25
|
end
|
25
26
|
|
26
27
|
# The current database adapter used by Tablature.
|
@@ -29,4 +30,10 @@ module Tablature
|
|
29
30
|
def self.database
|
30
31
|
configuration.database
|
31
32
|
end
|
33
|
+
|
34
|
+
class MissingPartition < StandardError
|
35
|
+
def initialize
|
36
|
+
super('Missing partition')
|
37
|
+
end
|
38
|
+
end
|
32
39
|
end
|
@@ -4,6 +4,7 @@ require_relative 'postgres/connection'
|
|
4
4
|
require_relative 'postgres/errors'
|
5
5
|
require_relative 'postgres/handlers/list'
|
6
6
|
require_relative 'postgres/handlers/range'
|
7
|
+
require_relative 'postgres/indexes'
|
7
8
|
require_relative 'postgres/partitioned_tables'
|
8
9
|
|
9
10
|
module Tablature
|
@@ -47,7 +48,7 @@ module Tablature
|
|
47
48
|
# @param [String, Symbol] table_name The name of the table to partition.
|
48
49
|
# @param [Hash] options The options to create the partition. Keys besides +:partition_key+
|
49
50
|
# will be passed to +create_table+.
|
50
|
-
# @option options [String, Symbol] :partition_key The partition key.
|
51
|
+
# @option options [String, Symbol, #call] :partition_key The partition key.
|
51
52
|
# @yield [td] A TableDefinition object. This allows creating the table columns the same way
|
52
53
|
# as Rails's +create_table+ does.
|
53
54
|
#
|
@@ -68,14 +69,44 @@ module Tablature
|
|
68
69
|
# @option options [String, Symbol] :values The values appearing in the partition.
|
69
70
|
# @option options [String, Symbol] :name The name of the partition. If it is not given, this
|
70
71
|
# will be randomly generated.
|
72
|
+
# @option options [Boolean] :default Whether the partition is the default partition or not.
|
71
73
|
#
|
72
74
|
# @example
|
73
75
|
# # With a table :events partitioned using the list method on the partition key `date`:
|
74
76
|
# create_list_partition_of :events, name: "events_2018-W49", values: [
|
75
|
-
# "2018-12-03", "2018-12-04", "2018-12-05", "2018-12-06", "2018-12-07", "2018-12-08",
|
77
|
+
# "2018-12-03", "2018-12-04", "2018-12-05", "2018-12-06", "2018-12-07", "2018-12-08",
|
78
|
+
# "2018-12-09"
|
76
79
|
# ]
|
77
80
|
delegate :create_list_partition_of, to: :list_handler
|
78
81
|
|
82
|
+
# @!method attach_to_list_partition(parent_table_name, options)
|
83
|
+
# Attaches a partition to a parent by specifying the key values appearing in the partition.
|
84
|
+
#
|
85
|
+
# @param parent_table_name [String, Symbol] The name of the parent table.
|
86
|
+
# @param [Hash] options The options to attach the partition.
|
87
|
+
# @option options [String, Symbol] :name The name of the partition.
|
88
|
+
# @option options [String, Symbol] :values The values appearing in the partition.
|
89
|
+
#
|
90
|
+
# @example
|
91
|
+
# # With a table :events partitioned using the list method on the partition key `date`:
|
92
|
+
# attach_to_list_partition :events, name: "events_2018-W49", values: [
|
93
|
+
# "2018-12-03", "2018-12-04", "2018-12-05", "2018-12-06", "2018-12-07", "2018-12-08",
|
94
|
+
# "2018-12-09"
|
95
|
+
# ]
|
96
|
+
delegate :attach_to_list_partition, to: :list_handler
|
97
|
+
|
98
|
+
# @!method detach_from_list_partition(parent_table_name, options)
|
99
|
+
# Detaches a partition from a parent.
|
100
|
+
#
|
101
|
+
# @param parent_table_name [String, Symbol] The name of the parent table.
|
102
|
+
# @param [Hash] options The options to create the partition.
|
103
|
+
# @option options [String, Symbol] :name The name of the partition.
|
104
|
+
#
|
105
|
+
# @example
|
106
|
+
# # With a table :events partitioned using the list method on the partition key `date`:
|
107
|
+
# detach_from_list_partition :events, name: "events_2018-W49"
|
108
|
+
delegate :detach_from_list_partition, to: :list_handler
|
109
|
+
|
79
110
|
# @!method create_range_partition(table_name, options, &block)
|
80
111
|
# Creates a partitioned table using the range partition method.
|
81
112
|
#
|
@@ -84,7 +115,6 @@ module Tablature
|
|
84
115
|
# @param [String, Symbol] table_name The name of the table to partition.
|
85
116
|
# @param [Hash] options The options to create the partition. Keys besides +:partition_key+
|
86
117
|
# will be passed to +create_table+.
|
87
|
-
# @option options [String, Symbol] :partition_key The partition key.
|
88
118
|
# @yield [td] A TableDefinition object. This allows creating the table columns the same way
|
89
119
|
# as Rails's +create_table+ does.
|
90
120
|
#
|
@@ -103,12 +133,13 @@ module Tablature
|
|
103
133
|
#
|
104
134
|
# @param parent_table_name [String, Symbol] The name of the parent table.
|
105
135
|
# @param [Hash] options The options to create the partition.
|
136
|
+
# @option options [String, Symbol] :name The name of the partition. If it is not given, this
|
137
|
+
# will be randomly generated.
|
106
138
|
# @option options [String, Symbol] :range_start The start of the range of values appearing in
|
107
139
|
# the partition.
|
108
140
|
# @option options [String, Symbol] :range_end The end of the range of values appearing in
|
109
141
|
# the partition.
|
110
|
-
# @option options [
|
111
|
-
# will be randomly generated.
|
142
|
+
# @option options [Boolean] :default Whether the partition is the default partition or not.
|
112
143
|
#
|
113
144
|
# @example
|
114
145
|
# # With a table :events partitioned using the range method on the partition key `date`:
|
@@ -116,6 +147,35 @@ module Tablature
|
|
116
147
|
# range_end: '2018-12-10'
|
117
148
|
delegate :create_range_partition_of, to: :range_handler
|
118
149
|
|
150
|
+
# @!method attach_to_range_partition(parent_table_name, options)
|
151
|
+
# Attaches a partition to a parent by specifying the key values appearing in the partition.
|
152
|
+
#
|
153
|
+
# @param parent_table_name [String, Symbol] The name of the parent table.
|
154
|
+
# @param [Hash] options The options to create the partition.
|
155
|
+
# @option options [String, Symbol] :name The name of the partition.
|
156
|
+
# @option options [String, Symbol] :range_start The start of the range of values appearing in
|
157
|
+
# the partition.
|
158
|
+
# @option options [String, Symbol] :range_end The end of the range of values appearing in
|
159
|
+
# the partition.
|
160
|
+
# @option options [Boolean] :default Whether the partition is the default partition or not.
|
161
|
+
#
|
162
|
+
# @example
|
163
|
+
# # With a table :events partitioned using the range method on the partition key `date`:
|
164
|
+
# attach_to_range_partition :events, name: "events_2018-W49", range_start: '2018-12-03',
|
165
|
+
# range_end: '2018-12-10'
|
166
|
+
delegate :attach_to_range_partition, to: :range_handler
|
167
|
+
|
168
|
+
# @!method detach_from_range_partition(parent_table_name, options)
|
169
|
+
# Detaches a partition from a parent.
|
170
|
+
#
|
171
|
+
# @param parent_table_name [String, Symbol] The name of the parent table.
|
172
|
+
# @param [Hash] options The options to detach the partition.
|
173
|
+
# @option options [String, Symbol] :name The name of the partition.
|
174
|
+
#
|
175
|
+
# @example
|
176
|
+
# detach_from_range_partition :events, name: "events_2018-W49"
|
177
|
+
delegate :detach_from_range_partition, to: :range_handler
|
178
|
+
|
119
179
|
# Returns an array of partitioned tables in the database.
|
120
180
|
#
|
121
181
|
# This collection of tables is used by the [Tablature::SchemaDumper] to populate the schema.rb
|
@@ -126,6 +186,17 @@ module Tablature
|
|
126
186
|
PartitionedTables.new(connection).all
|
127
187
|
end
|
128
188
|
|
189
|
+
# Indexes on the Partitioned Table.
|
190
|
+
#
|
191
|
+
# @param name [String] The name of the partitioned table we want indexes from.
|
192
|
+
# @return [Array<ActiveRecord::ConnectionAdapters::IndexDefinition>]
|
193
|
+
def indexes_on(partitioned_table)
|
194
|
+
return [] if Gem::Version.new(Rails.version) >= Gem::Version.new('6.0.3')
|
195
|
+
return [] unless connection.supports_indexes_on_partitioned_tables?
|
196
|
+
|
197
|
+
Indexes.new(connection).on(partitioned_table)
|
198
|
+
end
|
199
|
+
|
129
200
|
private
|
130
201
|
|
131
202
|
attr_reader :connectable
|
@@ -31,6 +31,20 @@ module Tablature
|
|
31
31
|
postgresql_version >= 110_000
|
32
32
|
end
|
33
33
|
|
34
|
+
# True if the connection supports default partitions.
|
35
|
+
#
|
36
|
+
# @return [Boolean]
|
37
|
+
def supports_default_partitions?
|
38
|
+
postgresql_version >= 110_000
|
39
|
+
end
|
40
|
+
|
41
|
+
# True if the connection supports indexes on partitioned tables.
|
42
|
+
#
|
43
|
+
# @return [Boolean]
|
44
|
+
def supports_indexes_on_partitioned_tables?
|
45
|
+
postgresql_version >= 110_000
|
46
|
+
end
|
47
|
+
|
34
48
|
# An integer representing the version of Postgres we're connected to.
|
35
49
|
#
|
36
50
|
# +postgresql_version+ is public in Rails 5, but protected in earlier
|
@@ -1,6 +1,23 @@
|
|
1
1
|
module Tablature
|
2
2
|
module Adapters
|
3
3
|
class Postgres
|
4
|
+
# Raised when a setting a partition as default on a database
|
5
|
+
# version that does not support default partitions.
|
6
|
+
#
|
7
|
+
# Default partitions are supported on Postgres 11 or newer.
|
8
|
+
class DefaultPartitionNotSupportedError < StandardError
|
9
|
+
def initialize
|
10
|
+
super('Default partitions require Postgres 11 or newer')
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Raised when trying to attach or detach a partition without supplying a name.
|
15
|
+
class MissingPartitionName < StandardError
|
16
|
+
def initialize
|
17
|
+
super('Missing partition name')
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
4
21
|
# Raised when a list partition operation is attempted on a database
|
5
22
|
# version that does not support list partitions.
|
6
23
|
#
|
@@ -15,7 +32,7 @@ module Tablature
|
|
15
32
|
# key.
|
16
33
|
class MissingListPartitionValuesError < StandardError
|
17
34
|
def initialize
|
18
|
-
super('Missing values for
|
35
|
+
super('Missing values for list partition')
|
19
36
|
end
|
20
37
|
end
|
21
38
|
|
@@ -33,7 +50,7 @@ module Tablature
|
|
33
50
|
# key.
|
34
51
|
class MissingRangePartitionBoundsError < StandardError
|
35
52
|
def initialize
|
36
|
-
super('Missing bounds for
|
53
|
+
super('Missing bounds for range partition')
|
37
54
|
end
|
38
55
|
end
|
39
56
|
end
|