full_search 0.4.2 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d5cf6b2007aa968f7243df89c2e957288336c03f64500278b88583e3a75035e
-  data.tar.gz: b575ebcb9d41e2a16fe3a86f5201705551c41f2066b4db80e31b2b15ecd2be09
+  metadata.gz: 99b631f72162065698bc187ad0b88cb7602cbead185b5c82209db713774e763d
+  data.tar.gz: 5e1dd5a0782e69cba81b0b2c8f32170046f4c0e3ad55debb063837117285a559
 SHA512:
-  metadata.gz: '0740089cb32424858a53942ab6695e7b4c65963a6cfa1ed75ee42aa500f702203c02f460d735d82f1fc9712e8df82f27f1e8b203700cb7e6be793641d4050821'
-  data.tar.gz: 02d341539689e75d8f2064fd457ae241f4008434804ba62931493c2bef64033197a968387b3ded9db17bbbfaa42687ae28249419a3ade9969ebee42aaced5e1c
+  metadata.gz: 146763f433fba78ed55b6a638b1d8a80bed0ea886daf294ff8e9389d09efef104b7ae6729d8897ee3ea267a4d38cf5d9956e691f4524da4a0861df6827c909ba
+  data.tar.gz: 65c5e59005e1730e12fadda9126b397a5fffdef462bcda088e7f14259b3635d1df7356d91dd23ebb312e6403489bf461b6088fc29a68b86b38fdbb80c4786ac2

data/lib/full_search/index.rb

@@ -29,6 +29,10 @@ module FullSearch
 
         create_metadata_table!
 
+        if table_exists?(model) && !healthy?(model)
+          rebuild!(model)
+        end
+
         fts_was_created = false
         unless table_exists?(model)
           conn.execute(create_virtual_table_sql(model))
@@ -67,8 +71,8 @@ module FullSearch
         with_rebuild_lock(model) do
           FullSearch::Instrumentation.instrument("rebuild", model: model.name) do
             drop_triggers!(model)
-            conn.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
-            conn.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
+            drop_fts_table_safely!(fts_table_name(model))
+            drop_fts_table_safely!(trigram_table_name(model))
             conn.execute(create_virtual_table_sql(model))
             if trigram_table_needed?(model)
               FullSearch::Typo.warn_unsupported!(model) unless FullSearch::Typo.supported?(model)
@@ -162,8 +166,14 @@ module FullSearch
         return unless table_exists?(model)
 
         conn = connection
-        conn.execute("DELETE FROM #{qt(fts_table_name(model))};")
-        conn.execute("DELETE FROM #{qt(trigram_table_name(model))};") if trigram_table_needed?(model) && trigram_table_exists?(model)
+        begin
+          conn.execute("DELETE FROM #{qt(fts_table_name(model))};")
+          conn.execute("DELETE FROM #{qt(trigram_table_name(model))};") if trigram_table_needed?(model) && trigram_table_exists?(model)
+        rescue ActiveRecord::StatementInvalid => e
+          raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
+
+          rebuild!(model)
+        end
       end
 
       def drop!(model)
@@ -171,8 +181,32 @@ module FullSearch
         verified_tables.delete(model.table_name)
         sqlite!(model)
         drop_triggers!(model)
-        connection.execute("DROP TABLE IF EXISTS #{qt(fts_table_name(model))};")
-        connection.execute("DROP TABLE IF EXISTS #{qt(trigram_table_name(model))};")
+        drop_fts_table_safely!(fts_table_name(model))
+        drop_fts_table_safely!(trigram_table_name(model))
+      end
+      # Drops an FTS5 virtual table, recovering from corruption where a bare
+      # DROP TABLE IF EXISTS raises "invalid fts5 file format" because the
+      # shadow tables were tampered with out-of-band. When that happens we
+      # drop the shadow tables directly, then drop the (now-empty) virtual
+      # table wrapper, so the caller can recreate a clean index.
+      def drop_fts_table_safely!(table_name)
+        connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};")
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
+
+        drop_shadow_tables!(table_name)
+        connection.execute("DROP TABLE IF EXISTS #{qt(table_name)};")
+      end
+
+      # FTS5 stores its index data in a fixed set of shadow tables named
+      # <table>_content, <table>_data, <table>_idx, <table>_docsize and
+      # <table>_config. Dropping these lets SQLite recover a virtual table
+      # that is otherwise undroppable due to internal corruption.
+      def drop_shadow_tables!(table_name)
+        %w[content data idx docsize config].each do |suffix|
+          shadow = "#{table_name}_#{suffix}"
+          connection.execute("DROP TABLE IF EXISTS #{qt(shadow)};")
+        end
       end
 
       def fts_table_name(model)
@@ -211,6 +245,17 @@ module FullSearch
         !table_exists?(model)
       end
 
+      def healthy?(model)
+        return false unless table_exists?(model)
+
+        connection.execute("SELECT count(*) FROM #{qt(fts_table_name(model))}")
+        true
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless e.message.match?(/fts5: corruption|invalid fts5 file format|malformed database schema/i)
+
+        false
+      end
+
       def trigram_table_needed?(model)
         model.full_search_dsl&.typo_tolerance? && model.full_search_dsl.tokenize != "trigram"
       end

data/lib/full_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FullSearch
-  VERSION = "0.4.2"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: full_search
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Ben D'Angelo
@@ -188,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '2.0'
 requirements: []
-rubygems_version: 4.0.15
+rubygems_version: 4.0.10
 specification_version: 4
 summary: SQLite FTS5 full-text search for Rails/ActiveRecord
 test_files: []