pg_party 1.1.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,32 +6,67 @@ module PgParty
6
6
  class Cache
7
7
  LOCK = Mutex.new
8
8
 
9
- class << self
10
- def clear!
11
- LOCK.synchronize { store.clear }
9
+ def initialize
10
+ # automatically initialize a new hash when
11
+ # accessing an object id that doesn't exist
12
+ @store = Hash.new { |h, k| h[k] = { models: {}, partitions: nil, partitions_with_subpartitions: nil } }
13
+ end
14
+
15
+ def clear!
16
+ LOCK.synchronize { @store.clear }
17
+
18
+ nil
19
+ end
20
+
21
+ def fetch_model(key, child_table, &block)
22
+ return block.call unless caching_enabled?
23
+
24
+ LOCK.synchronize { fetch_value(@store[key][:models], child_table.to_sym, block) }
25
+ end
26
+
27
+ def fetch_partitions(key, include_subpartitions, &block)
28
+ return block.call unless caching_enabled?
29
+ sub_key = include_subpartitions ? :partitions_with_subpartitions : :partitions
30
+
31
+ LOCK.synchronize { fetch_value(@store[key], sub_key, block) }
32
+ end
33
+
34
+ private
35
+
36
+ def caching_enabled?
37
+ PgParty.config.caching
38
+ end
39
+
40
+ def fetch_value(subhash, key, block)
41
+ entry = subhash[key]
12
42
 
13
- nil
43
+ if entry.nil? || entry.expired?
44
+ entry = Entry.new(block.call)
45
+ subhash[key] = entry
14
46
  end
15
47
 
16
- def fetch_model(key, child_table, &block)
17
- LOCK.synchronize do
18
- store[key][:models][child_table.to_sym] ||= block.call
19
- end
48
+ entry.value
49
+ end
50
+
51
+ class Entry
52
+ attr_reader :value
53
+
54
+ def initialize(value)
55
+ @value = value
56
+ @timestamp = Time.now
20
57
  end
21
58
 
22
- def fetch_partitions(key, &block)
23
- LOCK.synchronize do
24
- store[key][:partitions] ||= block.call
25
- end
59
+ def expired?
60
+ ttl.positive? && Time.now - @timestamp > ttl
26
61
  end
27
62
 
28
63
  private
29
64
 
30
- def store
31
- # automatically initialize a new hash when
32
- # accessing an object id that doesn't exist
33
- @store ||= Hash.new { |h, k| h[k] = { models: {}, partitions: nil } }
65
+ def ttl
66
+ PgParty.config.caching_ttl
34
67
  end
35
68
  end
69
+
70
+ private_constant :Entry
36
71
  end
37
72
  end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PgParty
4
+ class Config
5
+ attr_accessor \
6
+ :caching,
7
+ :caching_ttl,
8
+ :schema_exclude_partitions,
9
+ :create_template_tables,
10
+ :create_with_primary_key,
11
+ :include_subpartitions_in_partition_list
12
+
13
+ def initialize
14
+ @caching = true
15
+ @caching_ttl = -1
16
+ @schema_exclude_partitions = true
17
+ @create_template_tables = true
18
+ @create_with_primary_key = false
19
+ @include_subpartitions_in_partition_list = false
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PgParty
4
+ module Hacks
5
+ module PostgreSQLDatabaseTasks
6
+ def run_cmd(cmd, args, action)
7
+ if action != "dumping" || !PgParty.config.schema_exclude_partitions
8
+ return super
9
+ end
10
+
11
+ partitions = begin
12
+ ActiveRecord::Base.connection.select_values(
13
+ "SELECT DISTINCT inhrelid::regclass::text FROM pg_inherits"
14
+ )
15
+ rescue
16
+ []
17
+ end
18
+
19
+ excluded_tables = partitions.flat_map { |table| ["-T", "*.#{table}"] }
20
+
21
+ super(cmd, args + excluded_tables, action)
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pg_party/model_decorator"
4
+ require "ruby2_keywords"
5
+
6
+ module PgParty
7
+ module Model
8
+ module HashMethods
9
+ ruby2_keywords def create_partition(*args)
10
+ PgParty::ModelDecorator.new(self).create_hash_partition(*args)
11
+ end
12
+
13
+ ruby2_keywords def partition_key_in(*args)
14
+ PgParty::ModelDecorator.new(self).hash_partition_key_in(*args)
15
+ end
16
+ end
17
+ end
18
+ end
@@ -1,15 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "pg_party/model_decorator"
4
+ require "ruby2_keywords"
4
5
 
5
6
  module PgParty
6
7
  module Model
7
8
  module ListMethods
8
- def create_partition(*args)
9
+ ruby2_keywords def create_partition(*args)
9
10
  PgParty::ModelDecorator.new(self).create_list_partition(*args)
10
11
  end
11
12
 
12
- def partition_key_in(*args)
13
+ ruby2_keywords def create_default_partition(*args)
14
+ PgParty::ModelDecorator.new(self).create_default_partition(*args)
15
+ end
16
+
17
+ ruby2_keywords def partition_key_in(*args)
13
18
  PgParty::ModelDecorator.new(self).list_partition_key_in(*args)
14
19
  end
15
20
  end
@@ -5,12 +5,16 @@ require "pg_party/model_injector"
5
5
  module PgParty
6
6
  module Model
7
7
  module Methods
8
- def range_partition_by(key=nil, &blk)
9
- PgParty::ModelInjector.new(self, key || blk).inject_range_methods
8
+ def range_partition_by(*key, &blk)
9
+ PgParty::ModelInjector.new(self, *key, &blk).inject_range_methods
10
10
  end
11
11
 
12
- def list_partition_by(key=nil, &blk)
13
- PgParty::ModelInjector.new(self, key || blk).inject_list_methods
12
+ def list_partition_by(*key, &blk)
13
+ PgParty::ModelInjector.new(self, *key, &blk).inject_list_methods
14
+ end
15
+
16
+ def hash_partition_by(*key, &blk)
17
+ PgParty::ModelInjector.new(self, *key, &blk).inject_hash_methods
14
18
  end
15
19
 
16
20
  def partitioned?
@@ -1,15 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "pg_party/model_decorator"
4
+ require "ruby2_keywords"
4
5
 
5
6
  module PgParty
6
7
  module Model
7
8
  module RangeMethods
8
- def create_partition(*args)
9
+ ruby2_keywords def create_partition(*args)
9
10
  PgParty::ModelDecorator.new(self).create_range_partition(*args)
10
11
  end
11
12
 
12
- def partition_key_in(*args)
13
+ ruby2_keywords def create_default_partition(*args)
14
+ PgParty::ModelDecorator.new(self).create_default_partition(*args)
15
+ end
16
+
17
+ ruby2_keywords def partition_key_in(*args)
13
18
  PgParty::ModelDecorator.new(self).range_partition_key_in(*args)
14
19
  end
15
20
  end
@@ -6,15 +6,25 @@ module PgParty
6
6
  module Model
7
7
  module SharedMethods
8
8
  def reset_primary_key
9
- PgParty::ModelDecorator.new(self).partition_primary_key
9
+ return base_class.primary_key if self != base_class
10
+
11
+ partitions = partitions(include_subpartitions: PgParty.config.include_subpartitions_in_partition_list)
12
+ return get_primary_key(base_class.name) if partitions.empty?
13
+
14
+ first_partition = partitions.detect { |p| !connection.table_partitioned?(p) }
15
+ raise 'No leaf partitions exist for this model. Create a partition to contain your data' unless first_partition
16
+
17
+ in_partition(first_partition).get_primary_key(base_class.name)
10
18
  end
11
19
 
12
20
  def table_exists?
13
- PgParty::ModelDecorator.new(self).partition_table_exists?
21
+ target_table = partitions.first || table_name
22
+
23
+ connection.schema_cache.data_source_exists?(target_table)
14
24
  end
15
25
 
16
- def partitions
17
- PgParty::ModelDecorator.new(self).partitions
26
+ def partitions(**args)
27
+ PgParty::ModelDecorator.new(self).partitions(**args)
18
28
  end
19
29
 
20
30
  def in_partition(*args)
@@ -1,27 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "pg_party/cache"
4
-
5
3
  module PgParty
6
4
  class ModelDecorator < SimpleDelegator
7
- def partition_primary_key
8
- if self != base_class
9
- base_class.primary_key
10
- elsif partition_name = partitions.first
11
- in_partition(partition_name).get_primary_key(base_class.name)
12
- else
13
- get_primary_key(base_class.name)
14
- end
15
- end
16
-
17
- def partition_table_exists?
18
- target_table = partitions.first || table_name
19
-
20
- connection.schema_cache.data_source_exists?(target_table)
21
- end
22
-
23
5
  def in_partition(child_table_name)
24
- PgParty::Cache.fetch_model(cache_key, child_table_name) do
6
+ PgParty.cache.fetch_model(cache_key, child_table_name) do
25
7
  Class.new(__getobj__) do
26
8
  self.table_name = child_table_name
27
9
 
@@ -41,6 +23,11 @@ module PgParty
41
23
  def self.new(*args, &blk)
42
24
  superclass.new(*args, &blk)
43
25
  end
26
+
27
+ # to avoid unnecessary db lookups
28
+ def self.partitions
29
+ []
30
+ end
44
31
  end
45
32
  end
46
33
  end
@@ -49,7 +36,7 @@ module PgParty
49
36
  if complex_partition_key
50
37
  complex_partition_key_query("(#{partition_key}) = (?)", value)
51
38
  else
52
- where(current_arel_table[partition_key].eq(value))
39
+ where_partition_key(:eq, value)
53
40
  end
54
41
  end
55
42
 
@@ -61,9 +48,9 @@ module PgParty
61
48
  end_range
62
49
  )
63
50
  else
64
- node = current_arel_table[partition_key]
65
-
66
- where(node.gteq(start_range).and(node.lt(end_range)))
51
+ where_partition_key(:gteq, start_range).merge(
52
+ where_partition_key(:lt, end_range)
53
+ )
67
54
  end
68
55
  end
69
56
 
@@ -75,15 +62,11 @@ module PgParty
75
62
  end
76
63
  end
77
64
 
78
- def partitions
79
- PgParty::Cache.fetch_partitions(cache_key) do
80
- connection.select_values(<<-SQL)
81
- SELECT pg_inherits.inhrelid::regclass::text
82
- FROM pg_tables
83
- INNER JOIN pg_inherits
84
- ON pg_tables.tablename::regclass = pg_inherits.inhparent::regclass
85
- WHERE pg_tables.tablename = #{connection.quote(table_name)}
86
- SQL
65
+ alias_method :hash_partition_key_in, :list_partition_key_in
66
+
67
+ def partitions(include_subpartitions: PgParty.config.include_subpartitions_in_partition_list)
68
+ PgParty.cache.fetch_partitions(cache_key, include_subpartitions) do
69
+ connection.partitions_for_table_name(table_name, include_subpartitions: include_subpartitions)
87
70
  end
88
71
  rescue
89
72
  []
@@ -108,6 +91,23 @@ module PgParty
108
91
  create_partition(:create_list_partition_of, table_name, **modified_options)
109
92
  end
110
93
 
94
+ def create_hash_partition(modulus:, remainder:, **options)
95
+ modified_options = options.merge(
96
+ modulus: modulus,
97
+ remainder: remainder,
98
+ primary_key: primary_key,
99
+ )
100
+
101
+ create_partition(:create_hash_partition_of, table_name, **modified_options)
102
+ end
103
+
104
+ def create_default_partition(**options)
105
+ modified_options = options.merge(
106
+ primary_key: primary_key,
107
+ )
108
+ create_partition(:create_default_partition_of, table_name, **modified_options)
109
+ end
110
+
111
111
  private
112
112
 
113
113
  def create_partition(migration_method, table_name, **options)
@@ -145,5 +145,26 @@ module PgParty
145
145
 
146
146
  from(subquery, current_alias)
147
147
  end
148
+
149
+ def where_partition_key(meth, values)
150
+ partition_key_array = Array.wrap(partition_key)
151
+ values = Array.wrap(values)
152
+
153
+ if partition_key_array.size != values.size
154
+ raise "number of provided values does not match the number of partition key columns"
155
+ end
156
+
157
+ arel_query = partition_key_array.zip(values).inject(nil) do |obj, (column, value)|
158
+ node = current_arel_table[column].send(meth, value)
159
+
160
+ if obj.nil?
161
+ node
162
+ else
163
+ obj.and(node)
164
+ end
165
+ end
166
+
167
+ where(arel_query)
168
+ end
148
169
  end
149
170
  end
@@ -2,9 +2,10 @@
2
2
 
3
3
  module PgParty
4
4
  class ModelInjector
5
- def initialize(model, key)
5
+ def initialize(model, *key, &blk)
6
6
  @model = model
7
- @key = key
7
+ @key = key.flatten.compact
8
+ @key_blk = blk
8
9
  end
9
10
 
10
11
  def inject_range_methods
@@ -19,6 +20,12 @@ module PgParty
19
20
  inject_methods_for(PgParty::Model::ListMethods)
20
21
  end
21
22
 
23
+ def inject_hash_methods
24
+ require "pg_party/model/hash_methods"
25
+
26
+ inject_methods_for(PgParty::Model::HashMethods)
27
+ end
28
+
22
29
  private
23
30
 
24
31
  def inject_methods_for(mod)
@@ -38,11 +45,16 @@ module PgParty
38
45
  instance_predicate: false
39
46
  )
40
47
 
41
- if @key.is_a?(Proc)
42
- @model.partition_key = @key.call
48
+ if @key_blk
49
+ @model.partition_key = @key_blk.call
43
50
  @model.complex_partition_key = true
44
51
  else
45
- @model.partition_key = @key
52
+ if @key.size == 1
53
+ @model.partition_key = @key.first
54
+ else
55
+ @model.partition_key = @key
56
+ end
57
+
46
58
  @model.complex_partition_key = false
47
59
  end
48
60
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PgParty
4
- VERSION = "1.1.0"
4
+ VERSION = "1.5.0"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pg_party
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Krage
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-07 00:00:00.000000000 Z
11
+ date: 2021-01-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -16,20 +16,48 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '4.2'
19
+ version: '5.0'
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
- version: '6.1'
22
+ version: '6.2'
23
23
  type: :runtime
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: '4.2'
29
+ version: '5.0'
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
- version: '6.1'
32
+ version: '6.2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: ruby2_keywords
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: 0.0.2
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: 0.0.2
47
+ - !ruby/object:Gem::Dependency
48
+ name: parallel
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '1.0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1.0'
33
61
  - !ruby/object:Gem::Dependency
34
62
  name: appraisal
35
63
  requirement: !ruby/object:Gem::Requirement
@@ -64,14 +92,14 @@ dependencies:
64
92
  requirements:
65
93
  - - "~>"
66
94
  - !ruby/object:Gem::Version
67
- version: '1.1'
95
+ version: '1.3'
68
96
  type: :development
69
97
  prerelease: false
70
98
  version_requirements: !ruby/object:Gem::Requirement
71
99
  requirements:
72
100
  - - "~>"
73
101
  - !ruby/object:Gem::Version
74
- version: '1.1'
102
+ version: '1.3'
75
103
  - !ruby/object:Gem::Dependency
76
104
  name: database_cleaner
77
105
  requirement: !ruby/object:Gem::Requirement
@@ -182,14 +210,14 @@ dependencies:
182
210
  requirements:
183
211
  - - "~>"
184
212
  - !ruby/object:Gem::Version
185
- version: '0.17'
213
+ version: 0.17.0
186
214
  type: :development
187
215
  prerelease: false
188
216
  version_requirements: !ruby/object:Gem::Requirement
189
217
  requirements:
190
218
  - - "~>"
191
219
  - !ruby/object:Gem::Version
192
- version: '0.17'
220
+ version: 0.17.0
193
221
  - !ruby/object:Gem::Dependency
194
222
  name: timecop
195
223
  requirement: !ruby/object:Gem::Requirement
@@ -219,7 +247,9 @@ files:
219
247
  - lib/pg_party/adapter/postgresql_methods.rb
220
248
  - lib/pg_party/adapter_decorator.rb
221
249
  - lib/pg_party/cache.rb
222
- - lib/pg_party/hacks/schema_cache.rb
250
+ - lib/pg_party/config.rb
251
+ - lib/pg_party/hacks/postgresql_database_tasks.rb
252
+ - lib/pg_party/model/hash_methods.rb
223
253
  - lib/pg_party/model/list_methods.rb
224
254
  - lib/pg_party/model/methods.rb
225
255
  - lib/pg_party/model/range_methods.rb
@@ -239,14 +269,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
239
269
  requirements:
240
270
  - - ">="
241
271
  - !ruby/object:Gem::Version
242
- version: 2.3.0
272
+ version: 2.5.0
243
273
  required_rubygems_version: !ruby/object:Gem::Requirement
244
274
  requirements:
245
275
  - - ">="
246
276
  - !ruby/object:Gem::Version
247
277
  version: '0'
248
278
  requirements: []
249
- rubygems_version: 3.0.6
279
+ rubygems_version: 3.1.4
250
280
  signing_key:
251
281
  specification_version: 4
252
282
  summary: ActiveRecord PostgreSQL Partitioning