blind_index 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f656fa1765df9bf2bcfa9d994ccb9b5cf0504f27f7524d159323126462d973e0
4
- data.tar.gz: 88d5f2cd786f840e75540a204dbe7fbb74794de954978d90cdc69297078cf752
3
+ metadata.gz: '085bd0b1267657f9ff1d9cce4dac0f26e1c5359aa8d5d88969fd7b232da71094'
4
+ data.tar.gz: a0b856aaafbd264cffec4f0d2d88c51125e32b7e5959b8674ff43c721a7fb8c7
5
5
  SHA512:
6
- metadata.gz: 139d3d8f3aca413d0ee5045fe3212e6ed3327cdb6d0c60cb7eb2b314b4eb849abb42dcf17e386e23fb5521db2875b6c502b143fc46dc3305ad38151688b610be
7
- data.tar.gz: 0f5df7f99a1b79f2eab0bc9ff959b9fd21c238e772e09265ef88690678134f539968d9aebe508b88e9881657563013cd53031947d6a6322e08c7e29d95f45902
6
+ metadata.gz: 147b69fecf7b2366d7b0168019738823f093a3cae9cec6d0452abae2f24c8933b5864eea8d84b59d32ca85370c439612e8667e629690334f31c20f47fb732dba
7
+ data.tar.gz: 47c1ce7b7990e5abda27e9fa4991e5437219f645456f3ef4cfcf9891810179103d221eaba6adc8259147634bf702cd14e4313578177013ce422ebb29ba35946b
@@ -1,3 +1,7 @@
1
+ ## 2.0.1 (2020-02-14)
2
+
3
+ - Added `BlindIndex.backfill` method
4
+
1
5
  ## 2.0.0 (2019-02-10)
2
6
 
3
7
  - Blind indexes are updated immediately instead of in a `before_validation` callback
data/README.md CHANGED
@@ -60,10 +60,7 @@ end
60
60
  Backfill existing records
61
61
 
62
62
  ```ruby
63
- User.unscoped.where(email_bidx: nil).find_each do |user|
64
- user.compute_email_bidx
65
- user.save(validate: false)
66
- end
63
+ BlindIndex.backfill(User)
67
64
  ```
68
65
 
69
66
  And query away
@@ -115,10 +112,7 @@ end
115
112
  Backfill existing records
116
113
 
117
114
  ```ruby
118
- User.unscoped.where(email_ci_bidx: nil).find_each do |user|
119
- user.compute_email_ci_bidx
120
- user.save(validate: false)
121
- end
115
+ BlindIndex.backfill(User, columns: [:email_ci_bidx])
122
116
  ```
123
117
 
124
118
  And query away
@@ -168,10 +162,7 @@ end
168
162
  This allows you to backfill records while still querying the unencrypted field.
169
163
 
170
164
  ```ruby
171
- User.unscoped.where(email_bidx: nil).find_each do |user|
172
- user.compute_migrated_email_bidx
173
- user.save(validate: false)
174
- end
165
+ BlindIndex.backfill(User)
175
166
  ```
176
167
 
177
168
  Once that completes, you can remove the `migrating` option.
@@ -196,10 +187,7 @@ end
196
187
  This will keep the new column synced going forward. Next, backfill the data:
197
188
 
198
189
  ```ruby
199
- User.unscoped.where(email_bidx_v2: nil).find_each do |user|
200
- user.compute_rotated_email_bidx
201
- user.save(validate: false)
202
- end
190
+ BlindIndex.backfill(User, columns: [:email_bidx_v2])
203
191
  ```
204
192
 
205
193
  Then update your model
@@ -4,6 +4,7 @@ require "openssl"
4
4
  require "argon2/kdf"
5
5
 
6
6
  # modules
7
+ require "blind_index/backfill"
7
8
  require "blind_index/key_generator"
8
9
  require "blind_index/model"
9
10
  require "blind_index/version"
@@ -127,6 +128,10 @@ module BlindIndex
127
128
 
128
129
  key
129
130
  end
131
+
132
+ def self.backfill(relation, columns: nil, batch_size: 1000)
133
+ Backfill.new(relation, columns: columns, batch_size: batch_size).perform
134
+ end
130
135
  end
131
136
 
132
137
  ActiveSupport.on_load(:active_record) do
@@ -0,0 +1,110 @@
1
+ module BlindIndex
2
+ class Backfill
3
+ attr_reader :blind_indexes
4
+
5
+ def initialize(relation, batch_size:, columns:)
6
+ @relation = relation
7
+ @transaction = @relation.respond_to?(:transaction)
8
+ @batch_size = batch_size
9
+ @blind_indexes = @relation.blind_indexes
10
+ filter_columns!(columns) if columns
11
+ end
12
+
13
+ def perform
14
+ each_batch do |records|
15
+ backfill_records(records)
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ # modify in-place
22
+ def filter_columns!(columns)
23
+ columns = Array(columns).map(&:to_s)
24
+ blind_indexes.select! { |_, v| columns.include?(v[:bidx_attribute]) }
25
+ bad_columns = columns - blind_indexes.map { |_, v| v[:bidx_attribute] }
26
+ raise ArgumentError, "Bad column: #{bad_columns.first}" if bad_columns.any?
27
+ end
28
+
29
+ def build_relation
30
+ # build relation
31
+ relation = @relation
32
+
33
+ if defined?(ActiveRecord::Base) && relation.is_a?(ActiveRecord::Base)
34
+ relation = relation.unscoped
35
+ end
36
+
37
+ # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
38
+ relation = relation.all
39
+
40
+ attributes = blind_indexes.map { |_, v| v[:bidx_attribute] }
41
+
42
+ if defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation)
43
+ base_relation = relation.unscoped
44
+ or_relation = relation.unscoped
45
+
46
+ attributes.each_with_index do |attribute, i|
47
+ or_relation =
48
+ if i == 0
49
+ base_relation.where(attribute => nil)
50
+ else
51
+ or_relation.or(base_relation.where(attribute => nil))
52
+ end
53
+ end
54
+
55
+ relation.merge(or_relation)
56
+ else
57
+ relation.or(attributes.map { |a| {a => nil} })
58
+ end
59
+ end
60
+
61
+ def each_batch
62
+ relation = build_relation
63
+
64
+ if relation.respond_to?(:find_in_batches)
65
+ relation.find_in_batches(batch_size: @batch_size) do |records|
66
+ yield records
67
+ end
68
+ else
69
+ # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
70
+ # use cursor for Mongoid
71
+ records = []
72
+ relation.all.each do |record|
73
+ records << record
74
+ if records.length == @batch_size
75
+ yield records
76
+ records = []
77
+ end
78
+ end
79
+ yield records if records.any?
80
+ end
81
+ end
82
+
83
+ def backfill_records(records)
84
+ # do expensive blind index computation outside of transaction
85
+ records.each do |record|
86
+ blind_indexes.each do |k, v|
87
+ record.send("compute_#{k}_bidx") if !record.send(v[:bidx_attribute])
88
+ end
89
+ end
90
+
91
+ records.select! { |r| r.changed? }
92
+
93
+ with_transaction do
94
+ records.each do |record|
95
+ record.save!(validate: false)
96
+ end
97
+ end
98
+ end
99
+
100
+ def with_transaction
101
+ if @transaction
102
+ @relation.transaction do
103
+ yield
104
+ end
105
+ else
106
+ yield
107
+ end
108
+ end
109
+ end
110
+ end
@@ -65,7 +65,7 @@ module BlindIndex
65
65
  end
66
66
 
67
67
  define_method method_name do
68
- self.send("#{bidx_attribute}=", self.class.send(class_method_name, send(attribute)))
68
+ send("#{bidx_attribute}=", self.class.send(class_method_name, send(attribute)))
69
69
  end
70
70
 
71
71
  if callback
@@ -1,3 +1,3 @@
1
1
  module BlindIndex
2
- VERSION = "2.0.0"
2
+ VERSION = "2.0.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: blind_index
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-10 00:00:00.000000000 Z
11
+ date: 2020-02-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -174,6 +174,7 @@ files:
174
174
  - LICENSE.txt
175
175
  - README.md
176
176
  - lib/blind_index.rb
177
+ - lib/blind_index/backfill.rb
177
178
  - lib/blind_index/extensions.rb
178
179
  - lib/blind_index/key_generator.rb
179
180
  - lib/blind_index/model.rb