blind_index 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f656fa1765df9bf2bcfa9d994ccb9b5cf0504f27f7524d159323126462d973e0
-  data.tar.gz: 88d5f2cd786f840e75540a204dbe7fbb74794de954978d90cdc69297078cf752
+  metadata.gz: '085bd0b1267657f9ff1d9cce4dac0f26e1c5359aa8d5d88969fd7b232da71094'
+  data.tar.gz: a0b856aaafbd264cffec4f0d2d88c51125e32b7e5959b8674ff43c721a7fb8c7
 SHA512:
-  metadata.gz: 139d3d8f3aca413d0ee5045fe3212e6ed3327cdb6d0c60cb7eb2b314b4eb849abb42dcf17e386e23fb5521db2875b6c502b143fc46dc3305ad38151688b610be
-  data.tar.gz: 0f5df7f99a1b79f2eab0bc9ff959b9fd21c238e772e09265ef88690678134f539968d9aebe508b88e9881657563013cd53031947d6a6322e08c7e29d95f45902
+  metadata.gz: 147b69fecf7b2366d7b0168019738823f093a3cae9cec6d0452abae2f24c8933b5864eea8d84b59d32ca85370c439612e8667e629690334f31c20f47fb732dba
+  data.tar.gz: 47c1ce7b7990e5abda27e9fa4991e5437219f645456f3ef4cfcf9891810179103d221eaba6adc8259147634bf702cd14e4313578177013ce422ebb29ba35946b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.1 (2020-02-14)
+
+- Added `BlindIndex.backfill` method
+
 ## 2.0.0 (2019-02-10)
 
 - Blind indexes are updated immediately instead of in a `before_validation` callback

data/README.md

@@ -60,10 +60,7 @@ end
 Backfill existing records
 
 ```ruby
-User.unscoped.where(email_bidx: nil).find_each do |user|
-  user.compute_email_bidx
-  user.save(validate: false)
-end
+BlindIndex.backfill(User)
 ```
 
 And query away
@@ -115,10 +112,7 @@ end
 Backfill existing records
 
 ```ruby
-User.unscoped.where(email_ci_bidx: nil).find_each do |user|
-  user.compute_email_ci_bidx
-  user.save(validate: false)
-end
+BlindIndex.backfill(User, columns: [:email_ci_bidx])
 ```
 
 And query away
@@ -168,10 +162,7 @@ end
 This allows you to backfill records while still querying the unencrypted field.
 
 ```ruby
-User.unscoped.where(email_bidx: nil).find_each do |user|
-  user.compute_migrated_email_bidx
-  user.save(validate: false)
-end
+BlindIndex.backfill(User)
 ```
 
 Once that completes, you can remove the `migrating` option.
@@ -196,10 +187,7 @@ end
 This will keep the new column synced going forward. Next, backfill the data:
 
 ```ruby
-User.unscoped.where(email_bidx_v2: nil).find_each do |user|
-  user.compute_rotated_email_bidx
-  user.save(validate: false)
-end
+BlindIndex.backfill(User, columns: [:email_bidx_v2])
 ```
 
 Then update your model

data/lib/blind_index.rb

@@ -4,6 +4,7 @@ require "openssl"
 require "argon2/kdf"
 
 # modules
+require "blind_index/backfill"
 require "blind_index/key_generator"
 require "blind_index/model"
 require "blind_index/version"
@@ -127,6 +128,10 @@ module BlindIndex
 
     key
   end
+
+  def self.backfill(relation, columns: nil, batch_size: 1000)
+    Backfill.new(relation, columns: columns, batch_size: batch_size).perform
+  end
 end
 
 ActiveSupport.on_load(:active_record) do

data/lib/blind_index/backfill.rb

@@ -0,0 +1,110 @@
+module BlindIndex
+  class Backfill
+    attr_reader :blind_indexes
+
+    def initialize(relation, batch_size:, columns:)
+      @relation = relation
+      @transaction = @relation.respond_to?(:transaction)
+      @batch_size = batch_size
+      @blind_indexes = @relation.blind_indexes
+      filter_columns!(columns) if columns
+    end
+
+    def perform
+      each_batch do |records|
+        backfill_records(records)
+      end
+    end
+
+    private
+
+    # modify in-place
+    def filter_columns!(columns)
+      columns = Array(columns).map(&:to_s)
+      blind_indexes.select! { |_, v| columns.include?(v[:bidx_attribute]) }
+      bad_columns = columns - blind_indexes.map { |_, v| v[:bidx_attribute] }
+      raise ArgumentError, "Bad column: #{bad_columns.first}" if bad_columns.any?
+    end
+
+    def build_relation
+      # build relation
+      relation = @relation
+
+      if defined?(ActiveRecord::Base) && relation.is_a?(ActiveRecord::Base)
+        relation = relation.unscoped
+      end
+
+      # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
+      relation = relation.all
+
+      attributes = blind_indexes.map { |_, v| v[:bidx_attribute] }
+
+      if defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation)
+        base_relation = relation.unscoped
+        or_relation = relation.unscoped
+
+        attributes.each_with_index do |attribute, i|
+          or_relation =
+            if i == 0
+              base_relation.where(attribute => nil)
+            else
+              or_relation.or(base_relation.where(attribute => nil))
+            end
+        end
+
+        relation.merge(or_relation)
+      else
+        relation.or(attributes.map { |a| {a => nil} })
+      end
+    end
+
+    def each_batch
+      relation = build_relation
+
+      if relation.respond_to?(:find_in_batches)
+        relation.find_in_batches(batch_size: @batch_size) do |records|
+          yield records
+        end
+      else
+        # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
+        # use cursor for Mongoid
+        records = []
+        relation.all.each do |record|
+          records << record
+          if records.length == @batch_size
+            yield records
+            records = []
+          end
+        end
+        yield records if records.any?
+      end
+    end
+
+    def backfill_records(records)
+      # do expensive blind index computation outside of transaction
+      records.each do |record|
+        blind_indexes.each do |k, v|
+          record.send("compute_#{k}_bidx") if !record.send(v[:bidx_attribute])
+        end
+      end
+
+      records.select! { |r| r.changed? }
+
+      with_transaction do
+        records.each do |record|
+          record.save!(validate: false)
+        end
+      end
+    end
+
+    def with_transaction
+      if @transaction
+        @relation.transaction do
+          yield
+        end
+      else
+        yield
+      end
+    end
+  end
+end

data/lib/blind_index/model.rb

@@ -65,7 +65,7 @@ module BlindIndex
           end
 
           define_method method_name do
-            self.send("#{bidx_attribute}=", self.class.send(class_method_name, send(attribute)))
+            send("#{bidx_attribute}=", self.class.send(class_method_name, send(attribute)))
           end
 
           if callback

data/lib/blind_index/version.rb

@@ -1,3 +1,3 @@
 module BlindIndex
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blind_index
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-10 00:00:00.000000000 Z
+date: 2020-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -174,6 +174,7 @@ files:
 - LICENSE.txt
 - README.md
 - lib/blind_index.rb
+- lib/blind_index/backfill.rb
 - lib/blind_index/extensions.rb
 - lib/blind_index/key_generator.rb
 - lib/blind_index/model.rb