blind_index 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +4 -16
- data/lib/blind_index.rb +5 -0
- data/lib/blind_index/backfill.rb +110 -0
- data/lib/blind_index/model.rb +1 -1
- data/lib/blind_index/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '085bd0b1267657f9ff1d9cce4dac0f26e1c5359aa8d5d88969fd7b232da71094'
|
4
|
+
data.tar.gz: a0b856aaafbd264cffec4f0d2d88c51125e32b7e5959b8674ff43c721a7fb8c7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 147b69fecf7b2366d7b0168019738823f093a3cae9cec6d0452abae2f24c8933b5864eea8d84b59d32ca85370c439612e8667e629690334f31c20f47fb732dba
|
7
|
+
data.tar.gz: 47c1ce7b7990e5abda27e9fa4991e5437219f645456f3ef4cfcf9891810179103d221eaba6adc8259147634bf702cd14e4313578177013ce422ebb29ba35946b
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -60,10 +60,7 @@ end
|
|
60
60
|
Backfill existing records
|
61
61
|
|
62
62
|
```ruby
|
63
|
-
|
64
|
-
user.compute_email_bidx
|
65
|
-
user.save(validate: false)
|
66
|
-
end
|
63
|
+
BlindIndex.backfill(User)
|
67
64
|
```
|
68
65
|
|
69
66
|
And query away
|
@@ -115,10 +112,7 @@ end
|
|
115
112
|
Backfill existing records
|
116
113
|
|
117
114
|
```ruby
|
118
|
-
|
119
|
-
user.compute_email_ci_bidx
|
120
|
-
user.save(validate: false)
|
121
|
-
end
|
115
|
+
BlindIndex.backfill(User, columns: [:email_ci_bidx])
|
122
116
|
```
|
123
117
|
|
124
118
|
And query away
|
@@ -168,10 +162,7 @@ end
|
|
168
162
|
This allows you to backfill records while still querying the unencrypted field.
|
169
163
|
|
170
164
|
```ruby
|
171
|
-
|
172
|
-
user.compute_migrated_email_bidx
|
173
|
-
user.save(validate: false)
|
174
|
-
end
|
165
|
+
BlindIndex.backfill(User)
|
175
166
|
```
|
176
167
|
|
177
168
|
Once that completes, you can remove the `migrating` option.
|
@@ -196,10 +187,7 @@ end
|
|
196
187
|
This will keep the new column synced going forward. Next, backfill the data:
|
197
188
|
|
198
189
|
```ruby
|
199
|
-
|
200
|
-
user.compute_rotated_email_bidx
|
201
|
-
user.save(validate: false)
|
202
|
-
end
|
190
|
+
BlindIndex.backfill(User, columns: [:email_bidx_v2])
|
203
191
|
```
|
204
192
|
|
205
193
|
Then update your model
|
data/lib/blind_index.rb
CHANGED
@@ -4,6 +4,7 @@ require "openssl"
|
|
4
4
|
require "argon2/kdf"
|
5
5
|
|
6
6
|
# modules
|
7
|
+
require "blind_index/backfill"
|
7
8
|
require "blind_index/key_generator"
|
8
9
|
require "blind_index/model"
|
9
10
|
require "blind_index/version"
|
@@ -127,6 +128,10 @@ module BlindIndex
|
|
127
128
|
|
128
129
|
key
|
129
130
|
end
|
131
|
+
|
132
|
+
def self.backfill(relation, columns: nil, batch_size: 1000)
|
133
|
+
Backfill.new(relation, columns: columns, batch_size: batch_size).perform
|
134
|
+
end
|
130
135
|
end
|
131
136
|
|
132
137
|
ActiveSupport.on_load(:active_record) do
|
@@ -0,0 +1,110 @@
|
|
1
|
+
module BlindIndex
|
2
|
+
class Backfill
|
3
|
+
attr_reader :blind_indexes
|
4
|
+
|
5
|
+
def initialize(relation, batch_size:, columns:)
|
6
|
+
@relation = relation
|
7
|
+
@transaction = @relation.respond_to?(:transaction)
|
8
|
+
@batch_size = batch_size
|
9
|
+
@blind_indexes = @relation.blind_indexes
|
10
|
+
filter_columns!(columns) if columns
|
11
|
+
end
|
12
|
+
|
13
|
+
def perform
|
14
|
+
each_batch do |records|
|
15
|
+
backfill_records(records)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
# modify in-place
|
22
|
+
def filter_columns!(columns)
|
23
|
+
columns = Array(columns).map(&:to_s)
|
24
|
+
blind_indexes.select! { |_, v| columns.include?(v[:bidx_attribute]) }
|
25
|
+
bad_columns = columns - blind_indexes.map { |_, v| v[:bidx_attribute] }
|
26
|
+
raise ArgumentError, "Bad column: #{bad_columns.first}" if bad_columns.any?
|
27
|
+
end
|
28
|
+
|
29
|
+
def build_relation
|
30
|
+
# build relation
|
31
|
+
relation = @relation
|
32
|
+
|
33
|
+
if defined?(ActiveRecord::Base) && relation.is_a?(ActiveRecord::Base)
|
34
|
+
relation = relation.unscoped
|
35
|
+
end
|
36
|
+
|
37
|
+
# convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
|
38
|
+
relation = relation.all
|
39
|
+
|
40
|
+
attributes = blind_indexes.map { |_, v| v[:bidx_attribute] }
|
41
|
+
|
42
|
+
if defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation)
|
43
|
+
base_relation = relation.unscoped
|
44
|
+
or_relation = relation.unscoped
|
45
|
+
|
46
|
+
attributes.each_with_index do |attribute, i|
|
47
|
+
or_relation =
|
48
|
+
if i == 0
|
49
|
+
base_relation.where(attribute => nil)
|
50
|
+
else
|
51
|
+
or_relation.or(base_relation.where(attribute => nil))
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
relation.merge(or_relation)
|
56
|
+
else
|
57
|
+
relation.or(attributes.map { |a| {a => nil} })
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def each_batch
|
62
|
+
relation = build_relation
|
63
|
+
|
64
|
+
if relation.respond_to?(:find_in_batches)
|
65
|
+
relation.find_in_batches(batch_size: @batch_size) do |records|
|
66
|
+
yield records
|
67
|
+
end
|
68
|
+
else
|
69
|
+
# https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
|
70
|
+
# use cursor for Mongoid
|
71
|
+
records = []
|
72
|
+
relation.all.each do |record|
|
73
|
+
records << record
|
74
|
+
if records.length == @batch_size
|
75
|
+
yield records
|
76
|
+
records = []
|
77
|
+
end
|
78
|
+
end
|
79
|
+
yield records if records.any?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def backfill_records(records)
|
84
|
+
# do expensive blind index computation outside of transaction
|
85
|
+
records.each do |record|
|
86
|
+
blind_indexes.each do |k, v|
|
87
|
+
record.send("compute_#{k}_bidx") if !record.send(v[:bidx_attribute])
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
records.select! { |r| r.changed? }
|
92
|
+
|
93
|
+
with_transaction do
|
94
|
+
records.each do |record|
|
95
|
+
record.save!(validate: false)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def with_transaction
|
101
|
+
if @transaction
|
102
|
+
@relation.transaction do
|
103
|
+
yield
|
104
|
+
end
|
105
|
+
else
|
106
|
+
yield
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
data/lib/blind_index/model.rb
CHANGED
data/lib/blind_index/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: blind_index
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02-
|
11
|
+
date: 2020-02-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -174,6 +174,7 @@ files:
|
|
174
174
|
- LICENSE.txt
|
175
175
|
- README.md
|
176
176
|
- lib/blind_index.rb
|
177
|
+
- lib/blind_index/backfill.rb
|
177
178
|
- lib/blind_index/extensions.rb
|
178
179
|
- lib/blind_index/key_generator.rb
|
179
180
|
- lib/blind_index/model.rb
|