pg_party 1.0.1 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +297 -9
- data/lib/pg_party.rb +24 -3
- data/lib/pg_party/adapter/abstract_methods.rb +36 -0
- data/lib/pg_party/adapter/postgresql_methods.rb +45 -8
- data/lib/pg_party/adapter_decorator.rb +281 -27
- data/lib/pg_party/cache.rb +51 -16
- data/lib/pg_party/config.rb +22 -0
- data/lib/pg_party/hacks/postgresql_database_tasks.rb +25 -0
- data/lib/pg_party/model/hash_methods.rb +18 -0
- data/lib/pg_party/model/list_methods.rb +7 -2
- data/lib/pg_party/model/methods.rb +8 -4
- data/lib/pg_party/model/range_methods.rb +7 -2
- data/lib/pg_party/model/shared_methods.rb +14 -4
- data/lib/pg_party/model_decorator.rb +54 -42
- data/lib/pg_party/model_injector.rb +17 -5
- data/lib/pg_party/version.rb +1 -1
- metadata +57 -35
- data/lib/pg_party/hacks/schema_cache.rb +0 -13
data/lib/pg_party/cache.rb
CHANGED
@@ -6,32 +6,67 @@ module PgParty
|
|
6
6
|
class Cache
|
7
7
|
LOCK = Mutex.new
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
def initialize
|
10
|
+
# automatically initialize a new hash when
|
11
|
+
# accessing an object id that doesn't exist
|
12
|
+
@store = Hash.new { |h, k| h[k] = { models: {}, partitions: nil, partitions_with_subpartitions: nil } }
|
13
|
+
end
|
14
|
+
|
15
|
+
def clear!
|
16
|
+
LOCK.synchronize { @store.clear }
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def fetch_model(key, child_table, &block)
|
22
|
+
return block.call unless caching_enabled?
|
23
|
+
|
24
|
+
LOCK.synchronize { fetch_value(@store[key][:models], child_table.to_sym, block) }
|
25
|
+
end
|
26
|
+
|
27
|
+
def fetch_partitions(key, include_subpartitions, &block)
|
28
|
+
return block.call unless caching_enabled?
|
29
|
+
sub_key = include_subpartitions ? :partitions_with_subpartitions : :partitions
|
30
|
+
|
31
|
+
LOCK.synchronize { fetch_value(@store[key], sub_key, block) }
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def caching_enabled?
|
37
|
+
PgParty.config.caching
|
38
|
+
end
|
39
|
+
|
40
|
+
def fetch_value(subhash, key, block)
|
41
|
+
entry = subhash[key]
|
12
42
|
|
13
|
-
|
43
|
+
if entry.nil? || entry.expired?
|
44
|
+
entry = Entry.new(block.call)
|
45
|
+
subhash[key] = entry
|
14
46
|
end
|
15
47
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
48
|
+
entry.value
|
49
|
+
end
|
50
|
+
|
51
|
+
class Entry
|
52
|
+
attr_reader :value
|
53
|
+
|
54
|
+
def initialize(value)
|
55
|
+
@value = value
|
56
|
+
@timestamp = Time.now
|
20
57
|
end
|
21
58
|
|
22
|
-
def
|
23
|
-
|
24
|
-
store[key][:partitions] ||= block.call
|
25
|
-
end
|
59
|
+
def expired?
|
60
|
+
ttl.positive? && Time.now - @timestamp > ttl
|
26
61
|
end
|
27
62
|
|
28
63
|
private
|
29
64
|
|
30
|
-
def
|
31
|
-
|
32
|
-
# accessing an object id that doesn't exist
|
33
|
-
@store ||= Hash.new { |h, k| h[k] = { models: {}, partitions: nil } }
|
65
|
+
def ttl
|
66
|
+
PgParty.config.caching_ttl
|
34
67
|
end
|
35
68
|
end
|
69
|
+
|
70
|
+
private_constant :Entry
|
36
71
|
end
|
37
72
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PgParty
|
4
|
+
class Config
|
5
|
+
attr_accessor \
|
6
|
+
:caching,
|
7
|
+
:caching_ttl,
|
8
|
+
:schema_exclude_partitions,
|
9
|
+
:create_template_tables,
|
10
|
+
:create_with_primary_key,
|
11
|
+
:include_subpartitions_in_partition_list
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@caching = true
|
15
|
+
@caching_ttl = -1
|
16
|
+
@schema_exclude_partitions = true
|
17
|
+
@create_template_tables = true
|
18
|
+
@create_with_primary_key = false
|
19
|
+
@include_subpartitions_in_partition_list = false
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PgParty
|
4
|
+
module Hacks
|
5
|
+
module PostgreSQLDatabaseTasks
|
6
|
+
def run_cmd(cmd, args, action)
|
7
|
+
if action != "dumping" || !PgParty.config.schema_exclude_partitions
|
8
|
+
return super
|
9
|
+
end
|
10
|
+
|
11
|
+
partitions = begin
|
12
|
+
ActiveRecord::Base.connection.select_values(
|
13
|
+
"SELECT DISTINCT inhrelid::regclass::text FROM pg_inherits"
|
14
|
+
)
|
15
|
+
rescue
|
16
|
+
[]
|
17
|
+
end
|
18
|
+
|
19
|
+
excluded_tables = partitions.flat_map { |table| ["-T", "*.#{table}"] }
|
20
|
+
|
21
|
+
super(cmd, args + excluded_tables, action)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "pg_party/model_decorator"
|
4
|
+
require "ruby2_keywords"
|
5
|
+
|
6
|
+
module PgParty
|
7
|
+
module Model
|
8
|
+
module HashMethods
|
9
|
+
ruby2_keywords def create_partition(*args)
|
10
|
+
PgParty::ModelDecorator.new(self).create_hash_partition(*args)
|
11
|
+
end
|
12
|
+
|
13
|
+
ruby2_keywords def partition_key_in(*args)
|
14
|
+
PgParty::ModelDecorator.new(self).hash_partition_key_in(*args)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -1,15 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "pg_party/model_decorator"
|
4
|
+
require "ruby2_keywords"
|
4
5
|
|
5
6
|
module PgParty
|
6
7
|
module Model
|
7
8
|
module ListMethods
|
8
|
-
def create_partition(*args)
|
9
|
+
ruby2_keywords def create_partition(*args)
|
9
10
|
PgParty::ModelDecorator.new(self).create_list_partition(*args)
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
+
ruby2_keywords def create_default_partition(*args)
|
14
|
+
PgParty::ModelDecorator.new(self).create_default_partition(*args)
|
15
|
+
end
|
16
|
+
|
17
|
+
ruby2_keywords def partition_key_in(*args)
|
13
18
|
PgParty::ModelDecorator.new(self).list_partition_key_in(*args)
|
14
19
|
end
|
15
20
|
end
|
@@ -5,12 +5,16 @@ require "pg_party/model_injector"
|
|
5
5
|
module PgParty
|
6
6
|
module Model
|
7
7
|
module Methods
|
8
|
-
def range_partition_by(key
|
9
|
-
PgParty::ModelInjector.new(self, key
|
8
|
+
def range_partition_by(*key, &blk)
|
9
|
+
PgParty::ModelInjector.new(self, *key, &blk).inject_range_methods
|
10
10
|
end
|
11
11
|
|
12
|
-
def list_partition_by(key
|
13
|
-
PgParty::ModelInjector.new(self, key
|
12
|
+
def list_partition_by(*key, &blk)
|
13
|
+
PgParty::ModelInjector.new(self, *key, &blk).inject_list_methods
|
14
|
+
end
|
15
|
+
|
16
|
+
def hash_partition_by(*key, &blk)
|
17
|
+
PgParty::ModelInjector.new(self, *key, &blk).inject_hash_methods
|
14
18
|
end
|
15
19
|
|
16
20
|
def partitioned?
|
@@ -1,15 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "pg_party/model_decorator"
|
4
|
+
require "ruby2_keywords"
|
4
5
|
|
5
6
|
module PgParty
|
6
7
|
module Model
|
7
8
|
module RangeMethods
|
8
|
-
def create_partition(*args)
|
9
|
+
ruby2_keywords def create_partition(*args)
|
9
10
|
PgParty::ModelDecorator.new(self).create_range_partition(*args)
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
+
ruby2_keywords def create_default_partition(*args)
|
14
|
+
PgParty::ModelDecorator.new(self).create_default_partition(*args)
|
15
|
+
end
|
16
|
+
|
17
|
+
ruby2_keywords def partition_key_in(*args)
|
13
18
|
PgParty::ModelDecorator.new(self).range_partition_key_in(*args)
|
14
19
|
end
|
15
20
|
end
|
@@ -6,15 +6,25 @@ module PgParty
|
|
6
6
|
module Model
|
7
7
|
module SharedMethods
|
8
8
|
def reset_primary_key
|
9
|
-
|
9
|
+
return base_class.primary_key if self != base_class
|
10
|
+
|
11
|
+
partitions = partitions(include_subpartitions: true)
|
12
|
+
return get_primary_key(base_class.name) if partitions.empty?
|
13
|
+
|
14
|
+
first_partition = partitions.detect { |p| !connection.table_partitioned?(p) }
|
15
|
+
raise 'No leaf partitions exist for this model. Create a partition to contain your data' unless first_partition
|
16
|
+
|
17
|
+
in_partition(first_partition).get_primary_key(base_class.name)
|
10
18
|
end
|
11
19
|
|
12
20
|
def table_exists?
|
13
|
-
|
21
|
+
target_table = partitions.first || table_name
|
22
|
+
|
23
|
+
connection.schema_cache.data_source_exists?(target_table)
|
14
24
|
end
|
15
25
|
|
16
|
-
def partitions
|
17
|
-
PgParty::ModelDecorator.new(self).partitions
|
26
|
+
def partitions(*args)
|
27
|
+
PgParty::ModelDecorator.new(self).partitions(*args)
|
18
28
|
end
|
19
29
|
|
20
30
|
def in_partition(*args)
|
@@ -1,27 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "pg_party/cache"
|
4
|
-
|
5
3
|
module PgParty
|
6
4
|
class ModelDecorator < SimpleDelegator
|
7
|
-
def partition_primary_key
|
8
|
-
if self != base_class
|
9
|
-
base_class.primary_key
|
10
|
-
elsif partition_name = partitions.first
|
11
|
-
in_partition(partition_name).get_primary_key(base_class.name)
|
12
|
-
else
|
13
|
-
get_primary_key(base_class.name)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def partition_table_exists?
|
18
|
-
target_table = partitions.first || table_name
|
19
|
-
|
20
|
-
connection.schema_cache.data_source_exists?(target_table)
|
21
|
-
end
|
22
|
-
|
23
5
|
def in_partition(child_table_name)
|
24
|
-
PgParty
|
6
|
+
PgParty.cache.fetch_model(cache_key, child_table_name) do
|
25
7
|
Class.new(__getobj__) do
|
26
8
|
self.table_name = child_table_name
|
27
9
|
|
@@ -41,6 +23,11 @@ module PgParty
|
|
41
23
|
def self.new(*args, &blk)
|
42
24
|
superclass.new(*args, &blk)
|
43
25
|
end
|
26
|
+
|
27
|
+
# to avoid unnecessary db lookups
|
28
|
+
def self.partitions
|
29
|
+
[]
|
30
|
+
end
|
44
31
|
end
|
45
32
|
end
|
46
33
|
end
|
@@ -49,7 +36,7 @@ module PgParty
|
|
49
36
|
if complex_partition_key
|
50
37
|
complex_partition_key_query("(#{partition_key}) = (?)", value)
|
51
38
|
else
|
52
|
-
|
39
|
+
where_partition_key(:eq, value)
|
53
40
|
end
|
54
41
|
end
|
55
42
|
|
@@ -61,9 +48,9 @@ module PgParty
|
|
61
48
|
end_range
|
62
49
|
)
|
63
50
|
else
|
64
|
-
|
65
|
-
|
66
|
-
|
51
|
+
where_partition_key(:gteq, start_range).merge(
|
52
|
+
where_partition_key(:lt, end_range)
|
53
|
+
)
|
67
54
|
end
|
68
55
|
end
|
69
56
|
|
@@ -75,15 +62,11 @@ module PgParty
|
|
75
62
|
end
|
76
63
|
end
|
77
64
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
INNER JOIN pg_inherits
|
84
|
-
ON pg_tables.tablename::regclass = pg_inherits.inhparent::regclass
|
85
|
-
WHERE pg_tables.tablename = #{connection.quote(table_name)}
|
86
|
-
SQL
|
65
|
+
alias_method :hash_partition_key_in, :list_partition_key_in
|
66
|
+
|
67
|
+
def partitions(include_subpartitions: PgParty.config.include_subpartitions_in_partition_list)
|
68
|
+
PgParty.cache.fetch_partitions(cache_key, include_subpartitions) do
|
69
|
+
connection.partitions_for_table_name(table_name, include_subpartitions: include_subpartitions)
|
87
70
|
end
|
88
71
|
rescue
|
89
72
|
[]
|
@@ -108,6 +91,23 @@ module PgParty
|
|
108
91
|
create_partition(:create_list_partition_of, table_name, **modified_options)
|
109
92
|
end
|
110
93
|
|
94
|
+
def create_hash_partition(modulus:, remainder:, **options)
|
95
|
+
modified_options = options.merge(
|
96
|
+
modulus: modulus,
|
97
|
+
remainder: remainder,
|
98
|
+
primary_key: primary_key,
|
99
|
+
)
|
100
|
+
|
101
|
+
create_partition(:create_hash_partition_of, table_name, **modified_options)
|
102
|
+
end
|
103
|
+
|
104
|
+
def create_default_partition(**options)
|
105
|
+
modified_options = options.merge(
|
106
|
+
primary_key: primary_key,
|
107
|
+
)
|
108
|
+
create_partition(:create_default_partition_of, table_name, **modified_options)
|
109
|
+
end
|
110
|
+
|
111
111
|
private
|
112
112
|
|
113
113
|
def create_partition(migration_method, table_name, **options)
|
@@ -138,21 +138,33 @@ module PgParty
|
|
138
138
|
end
|
139
139
|
end
|
140
140
|
|
141
|
-
def model_class
|
142
|
-
if respond_to?(:klass)
|
143
|
-
klass
|
144
|
-
else
|
145
|
-
self
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
141
|
def complex_partition_key_query(clause, *interpolated_values)
|
150
|
-
subquery =
|
151
|
-
.unscoped
|
142
|
+
subquery = unscoped
|
152
143
|
.select("*")
|
153
144
|
.where(clause, *interpolated_values)
|
154
145
|
|
155
146
|
from(subquery, current_alias)
|
156
147
|
end
|
148
|
+
|
149
|
+
def where_partition_key(meth, values)
|
150
|
+
partition_key_array = Array.wrap(partition_key)
|
151
|
+
values = Array.wrap(values)
|
152
|
+
|
153
|
+
if partition_key_array.size != values.size
|
154
|
+
raise "number of provided values does not match the number of partition key columns"
|
155
|
+
end
|
156
|
+
|
157
|
+
arel_query = partition_key_array.zip(values).inject(nil) do |obj, (column, value)|
|
158
|
+
node = current_arel_table[column].send(meth, value)
|
159
|
+
|
160
|
+
if obj.nil?
|
161
|
+
node
|
162
|
+
else
|
163
|
+
obj.and(node)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
where(arel_query)
|
168
|
+
end
|
157
169
|
end
|
158
170
|
end
|
@@ -2,9 +2,10 @@
|
|
2
2
|
|
3
3
|
module PgParty
|
4
4
|
class ModelInjector
|
5
|
-
def initialize(model, key)
|
5
|
+
def initialize(model, *key, &blk)
|
6
6
|
@model = model
|
7
|
-
@key = key
|
7
|
+
@key = key.flatten.compact
|
8
|
+
@key_blk = blk
|
8
9
|
end
|
9
10
|
|
10
11
|
def inject_range_methods
|
@@ -19,6 +20,12 @@ module PgParty
|
|
19
20
|
inject_methods_for(PgParty::Model::ListMethods)
|
20
21
|
end
|
21
22
|
|
23
|
+
def inject_hash_methods
|
24
|
+
require "pg_party/model/hash_methods"
|
25
|
+
|
26
|
+
inject_methods_for(PgParty::Model::HashMethods)
|
27
|
+
end
|
28
|
+
|
22
29
|
private
|
23
30
|
|
24
31
|
def inject_methods_for(mod)
|
@@ -38,11 +45,16 @@ module PgParty
|
|
38
45
|
instance_predicate: false
|
39
46
|
)
|
40
47
|
|
41
|
-
if @
|
42
|
-
@model.partition_key = @
|
48
|
+
if @key_blk
|
49
|
+
@model.partition_key = @key_blk.call
|
43
50
|
@model.complex_partition_key = true
|
44
51
|
else
|
45
|
-
@
|
52
|
+
if @key.size == 1
|
53
|
+
@model.partition_key = @key.first
|
54
|
+
else
|
55
|
+
@model.partition_key = @key
|
56
|
+
end
|
57
|
+
|
46
58
|
@model.complex_partition_key = false
|
47
59
|
end
|
48
60
|
end
|
data/lib/pg_party/version.rb
CHANGED