litestack 0.4.1 → 0.4.3

data/lib/litestack/litesearch/index.rb

@@ -1,8 +1,7 @@
-require 'oj'
-require_relative './schema.rb'
+require "oj"
+require_relative "./schema"
 
 class Litesearch::Index
-  
   DEFAULT_SEARCH_OPTIONS = {limit: 25, offset: 0}
 
   def initialize(db, name)
@@ -17,14 +16,14 @@ class Litesearch::Index
     # if they differ in tokenizer then rebuild if auto-rebuild is on (error otherwise)
     # if they differ in both then update the structure and rebuild if auto-rebuild is on (error otherwise)
     load_index(name) if exists?(name)
-    
+
     if block_given?
       schema = Litesearch::Schema.new
       schema.schema[:name] = name
       yield schema
       schema.post_init
       # now that we have a schema object we need to check if we need to create or modify and existing index
-      @db.transaction(:immediate) do
+      if @db.transaction_active?
         if exists?(name)
           load_index(name)
           do_modify(schema)
@@ -32,111 +31,141 @@ class Litesearch::Index
           do_create(schema)
         end
         prepare_statements
-      end
-    else
-      if exists?(name)
-        # an index already exists, load it from the database and return the index instance to the caller
-        load_index(name)
-        prepare_statements
       else
-        raise "index does not exist and no schema was supplied"
+        @db.transaction(:immediate) do
+          if exists?(name)
+            load_index(name)
+            do_modify(schema)
+          else
+            do_create(schema)
+          end
+          prepare_statements
+        end
       end
+    elsif exists?(name)
+      load_index(name)
+      prepare_statements
+    # an index already exists, load it from the database and return the index instance to the caller
+    else
+      raise "index does not exist and no schema was supplied"
     end
   end
-  
+
   def load_index(name)
     # we cannot use get_config_value here since the schema object is not created yet, should we allow something here?
-    @schema = Litesearch::Schema.new(Oj.load(@db.get_first_value("SELECT v from #{name}_config where k = ?", :litesearch_schema.to_s))) rescue nil
+    @schema = begin
+      Litesearch::Schema.new(Oj.load(@db.get_first_value("SELECT v from #{name}_config where k = ?", :litesearch_schema.to_s)))
+    rescue
+      nil
+    end
     raise "index configuration not found, either corrupted or not a litesearch index!" if @schema.nil?
     self
   end
-          
+
   def modify
     schema = Litesearch::Schema.new
     yield schema
     schema.schema[:name] = @schema.schema[:name]
     do_modify(schema)
   end
-  
+
   def rebuild!
-    @db.transaction(:immediate) do
-      do_rebuild
+    if @db.transaction_active?
+        do_rebuild
+    else
+      @db.transaction(:immediate) { do_rebuild }
     end
   end
-  
+
   def add(document)
     @stmts[:insert].execute!(document)
-    return @db.last_insert_row_id
+    @db.last_insert_row_id
   end
-  
+
   def remove(id)
     @stmts[:delete].execute!(id)
   end
-  
+
   def count(term = nil)
     if term
       @stmts[:count].execute!(term)[0][0]
     else
-      @stmts[:count_all].execute!()[0][0]
+      @stmts[:count_all].execute![0][0]
     end
   end
-  
+
   # search options include
   # limit: how many records to return
   # offset: start from which record
   def search(term, options = {})
-    result = []
     options = DEFAULT_SEARCH_OPTIONS.merge(options)
     rs = @stmts[:search].execute(term, options[:limit], options[:offset])
-    if @db.results_as_hash
-      rs.each_hash do |hash|
-        result << hash 
-      end
+    generate_results(rs)
+  end
+  
+  def similar(id, limit=10)
+    #   pp term = @db.execute(@schema.sql_for(:similarity_query), id)
+    if @schema.schema[:tokenizer] == :trigram
+      # just use the normal similarity approach for now
+      # need to recondisder that for trigram indexes later
+      rs = @stmts[:similar].execute(id, limit)
     else
-      result = rs.to_a
-    end    
-    result
+      rs = @stmts[:similar].execute(id, limit)
+    end
+    generate_results(rs)
   end
-      
+  
   def clear!
     @stmts[:delete_all].execute!(id)
   end
-  
+
   def drop!
     if @schema.get(:type) == :backed
       @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
-      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+      if @schema.sql_for(:create_secondary_triggers)
         @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
       end
     end
     @db.execute(@schema.sql_for(:drop))
   end
-  
-  
+
   private
-  
+
+  def generate_results(rs)
+    result = []
+    if @db.results_as_hash
+      rs.each_hash do |hash|
+        result << hash
+      end
+    else
+      result = rs.to_a
+    end
+    result    
+  end
+
   def exists?(name)
     @db.get_first_value("SELECT count(*) FROM SQLITE_MASTER WHERE name = ? AND type = 'table' AND (sql like '%fts5%' OR sql like '%FTS5%')", name.to_s) == 1
   end
 
   def prepare_statements
-    stmt_names = [:insert, :delete, :delete_all, :drop, :count, :count_all, :search]
+    stmt_names = [:insert, :delete, :delete_all, :drop, :count, :count_all, :search, :similar]
     stmt_names.each do |stmt_name|
       @stmts[stmt_name] = @db.prepare(@schema.sql_for(stmt_name))
-    end      
+    end
   end
-  
-  def do_create(schema)   
-    @schema = schema   
+
+  def do_create(schema)
+    @schema = schema
     @schema.clean
     # create index
     @db.execute(schema.sql_for(:create_index, true))
+    @db.execute_batch(schema.sql_for(:create_vocab_tables))
     # adjust ranking function
     @db.execute(schema.sql_for(:ranks, true))
     # create triggers (if any)
     if @schema.get(:type) == :backed
       @db.execute_batch(@schema.sql_for(:create_primary_triggers))
-      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+      if (secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers))
         @db.execute_batch(secondary_triggers_sql)
       end
       @db.execute(@schema.sql_for(:rebuild)) if @schema.get(:rebuild_on_create)
@@ -152,11 +181,11 @@ class Litesearch::Index
     requires_schema_change = false
     requires_trigger_change = false
     requires_rebuild = false
-    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:filter_column] || changes[:removed_fields_count] > 0# any change here will require a schema change
+    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:filter_column] || changes[:removed_fields_count] > 0 # any change here will require a schema change
       requires_schema_change = true
       # only a change in tokenizer
       requires_rebuild = changes[:tokenizer] || new_schema.get(:rebuild_on_modify)
-      requires_trigger_change = (changes[:table] || changes[:fields] || changes[:filter_column]) && @schema.get(:type) == :backed 
+      requires_trigger_change = (changes[:table] || changes[:fields] || changes[:filter_column]) && @schema.get(:type) == :backed
     end
     if requires_schema_change
       # 1. enable schema editing
@@ -169,12 +198,12 @@ class Litesearch::Index
       @db.execute(new_schema.sql_for(:expand_data), changes[:extra_fields_count])
       @db.execute(new_schema.sql_for(:expand_docsize), changes[:extra_fields_count])
       @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
-      # need to reprepare statements        
+      # need to reprepare statements
     end
     if requires_trigger_change
       @db.execute_batch(new_schema.sql_for(:drop_primary_triggers))
       @db.execute_batch(new_schema.sql_for(:create_primary_triggers))
-      if secondary_triggers_sql = new_schema.sql_for(:create_secondary_triggers)
+      if (secondary_triggers_sql = new_schema.sql_for(:create_secondary_triggers))
         @db.execute_batch(new_schema.sql_for(:drop_secondary_triggers))
         @db.execute_batch(secondary_triggers_sql)
       end
@@ -183,48 +212,49 @@ class Litesearch::Index
       @schema = new_schema
       set_config_value(:litesearch_schema, @schema.schema)
       prepare_statements
-      #save_schema
+      # save_schema
     end
+    # update the weights if they changed
+    @db.execute(@schema.sql_for(:ranks, true)) if changes[:weights]
+    @db.execute_batch(@schema.sql_for(:create_vocab_tables))
     do_rebuild if requires_rebuild
-    # update the weights if they changed 
-    @db.execute(@schema.sql_for(:ranks)) if changes[:weights]
   end
 
   def do_rebuild
     # remove any zero weight columns
     if @schema.get(:type) == :backed
       @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
-      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+      if (secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers))
         @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
       end
       @db.execute(@schema.sql_for(:drop))
       @db.execute(@schema.sql_for(:create_index, true))
       @db.execute_batch(@schema.sql_for(:create_primary_triggers))
       @db.execute_batch(secondary_triggers_sql) if secondary_triggers_sql
-      @db.execute(@schema.sql_for(:rebuild)) 
+      @db.execute(@schema.sql_for(:rebuild))
     elsif @schema.get(:type) == :standalone
       removables = []
-      @schema.get(:fields).each_with_index{|f, i| removables << [f[0], i] if f[1][:weight] == 0  }
+      @schema.get(:fields).each_with_index { |f, i| removables << [f[0], i] if f[1][:weight] == 0 }
       removables.each do |col|
         @db.execute(@schema.sql_for(:drop_content_col, col[1]))
-        @schema.get(:fields).delete(col[0]) 
+        @schema.get(:fields).delete(col[0])
       end
       @db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
       @db.execute(@schema.sql_for(:update_index), @schema.sql_for(:create_index, true))
-      @db.execute(@schema.sql_for(:update_content_table), @schema.sql_for(:create_content_table, @schema.schema[:fields].count)) 
+      @db.execute(@schema.sql_for(:update_content_table), @schema.sql_for(:create_content_table, @schema.schema[:fields].count))
       @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
-      @db.execute(@schema.sql_for(:rebuild)) 
+      @db.execute(@schema.sql_for(:rebuild))
     end
+    @db.execute_batch(@schema.sql_for(:create_vocab_tables))
     set_config_value(:litesearch_schema, @schema.schema)
-    @db.execute(@schema.sql_for(:ranks, true)) 
- end
-  
+    @db.execute(@schema.sql_for(:ranks, true))
+  end
+
   def get_config_value(key)
-    Oj.load(@db.get_first_value(@schema.sql_for(:get_config_value), key.to_s)) #rescue nil
+    Oj.load(@db.get_first_value(@schema.sql_for(:get_config_value), key.to_s)) # rescue nil
   end
-  
+
   def set_config_value(key, value)
     @db.execute(@schema.sql_for(:set_config_value), key.to_s, Oj.dump(value))
   end
-
 end

data/lib/litestack/litesearch/model.rb

@@ -1,61 +1,75 @@
 module Litesearch::Model
-                      
   def self.included(klass)
     klass.include InstanceMethods
-    klass.extend ClassMethods 
+    klass.extend ClassMethods
     klass.attribute :search_rank, :float if klass.respond_to? :attribute
-    if defined?(Sequel::Model) != nil && klass.ancestors.include?(Sequel::Model)
-       klass.include Litesearch::Model::SequelInstanceMethods
-       klass.extend Litesearch::Model::SequelClassMethods
-       Sequel::Model.extend Litesearch::Model::BaseClassMethods
-    elsif defined?(ActiveRecord::Base) != nil && klass.ancestors.include?(ActiveRecord::Base)
-       klass.include Litesearch::Model::ActiveRecordInstanceMethods
-       klass.extend Litesearch::Model::ActiveRecordClassMethods
-       ActiveRecord::Base.extend Litesearch::Model::BaseClassMethods
+    if !defined?(Sequel::Model).nil? && klass.ancestors.include?(Sequel::Model)
+      klass.include Litesearch::Model::SequelInstanceMethods
+      klass.extend Litesearch::Model::SequelClassMethods
+      Sequel::Model.extend Litesearch::Model::BaseClassMethods
+    elsif !defined?(ActiveRecord::Base).nil? && klass.ancestors.include?(ActiveRecord::Base)
+      klass.include Litesearch::Model::ActiveRecordInstanceMethods
+      klass.extend Litesearch::Model::ActiveRecordClassMethods
+      ActiveRecord::Base.extend Litesearch::Model::BaseClassMethods
     end
   end
-  
+
   module BaseClassMethods
     def search_models
       @@models ||= {}
-    end 
+    end
   end
-  
+
   module InstanceMethods
-  
+    def similar(limit=10)
+      conn = self.class.get_connection
+      idx = conn.search_index(self.class.send(:index_name))
+      r_a_h = conn.results_as_hash
+      conn.results_as_hash = true
+      rs = idx.similar(id, limit)
+      conn.results_as_hash = r_a_h
+      result = []
+      rs.each do |row|
+        obj = self.class.fetch_row(row["id"])
+        obj.search_rank = row["search_rank"]
+        result << obj
+      end
+      result
+    end
+        
   end
-  
-  module ClassMethods  
-     
+
+  module ClassMethods
     def litesearch
       idx = get_connection.search_index(index_name) do |schema|
         schema.type :backed
         schema.table table_name.to_sym
         yield schema
         schema.post_init
-        @schema = schema #save the schema   
+        @schema = schema # save the schema
       end
-      if defined?(Sequel::Model) != nil && self.ancestors.include?(Sequel::Model)
-        Sequel::Model.search_models[self.name] = self
-      elsif defined?(ActiveRecord::Base) != nil && self.ancestors.include?(ActiveRecord::Base)
-        ActiveRecord::Base.search_models[self.name] = self
+      if !defined?(Sequel::Model).nil? && ancestors.include?(Sequel::Model)
+        Sequel::Model.search_models[name] = self
+      elsif !defined?(ActiveRecord::Base).nil? && ancestors.include?(ActiveRecord::Base)
+        ActiveRecord::Base.search_models[name] = self
       end
       idx
     end
-    
+
     def rebuild_index!
       get_connection.search_index(index_name).rebuild!
     end
-    
+
     def drop_index!
       get_connection.search_index(index_name).drop!
     end
-    
-    def search_all(term, options={})
+
+    def search_all(term, options = {})
       options[:offset] ||= 0
       options[:limit] ||= 25
-      selects = [] 
-      if models = options[:models]
+      options[:term] = term
+      selects = []
+      if (models = options[:models])
         models_hash = {}
         models.each do |model|
           models_hash[model.name] = model
@@ -63,13 +77,15 @@ module Litesearch::Model
       else
         models_hash = search_models
       end
+      # remove the models from the options hash before passing it ot the query
+      options.delete(:models)
       models_hash.each do |name, klass|
-        selects << "SELECT '#{name}' AS model, rowid, -rank AS search_rank FROM #{index_name_for_table(klass.table_name)}('#{term}')"
+        selects << "SELECT '#{name}' AS model, rowid, -rank AS search_rank FROM #{index_name_for_table(klass.table_name)}(:term)"
       end
       conn = get_connection
-      sql = selects.join(" UNION ") << " ORDER BY search_rank DESC LIMIT #{options[:limit]} OFFSET #{options[:offset]}"
+      sql = selects.join(" UNION ") << " ORDER BY search_rank DESC LIMIT :limit OFFSET :offset"
       result = []
-      rs = conn.query(sql) #, options[:limit], options[:offset])
+      rs = conn.query(sql, options) # , options[:limit], options[:offset])
       rs.each_hash do |row|
         obj = models_hash[row["model"]].fetch_row(row["rowid"])
         obj.search_rank = row["search_rank"]
@@ -78,39 +94,32 @@ module Litesearch::Model
       rs.close
       result
     end
-          
-    # AR specific
-                     
-    private
-    
+
     def index_name
       "#{table_name}_search_idx"
     end
-    
+
     def index_name_for_table(table)
       "#{table}_search_idx"
     end
-    
+
     # create a new instance of self with the row as an argument
     def create_instance(row)
-      self.new(row)
+      new(row)
     end
-    
-          
   end
-  
-  module ActiveRecordInstanceMethods;end
-    
+
+  module ActiveRecordInstanceMethods; end
+
   module ActiveRecordClassMethods
-    
     def get_connection
       connection.raw_connection
     end
-    
+
     def fetch_row(id)
       find(id)
     end
-    
+
     def search(term)
       self.select(
         "#{table_name}.*"
@@ -120,30 +129,25 @@ module Litesearch::Model
         "-#{index_name}.rank AS search_rank"
       ).order(
         Arel.sql("#{index_name}.rank")
-      )        
+      )
     end
 
-    private
-    
     def create_instance(row)
       instantiate(row)
     end
   end
-  
+
   module SequelInstanceMethods
-  
     def search_rank
       @values[:search_rank]
     end
-    
+
     def search_rank=(rank)
       @values[:search_rank] = rank
     end
-        
   end
-  
+
   module SequelClassMethods
-        
     def fetch_row(id)
       self[id]
     end
@@ -156,23 +160,20 @@ module Litesearch::Model
       dataset.select(
         Sequel.lit("#{table_name}.*, -#{index_name}.rank AS search_rank")
       ).inner_join(
-        Sequel.lit("#{index_name}('#{term}') ON #{table_name}.id = #{index_name}.rowid AND rank != 0")
+        Sequel.lit("#{index_name}(:term) ON #{table_name}.id = #{index_name}.rowid AND rank != 0", {term: term})
       ).order(
-        Sequel.lit('rank')
+        Sequel.lit("rank")
       )
     end
 
-    private
-    
     def create_instance(row)
       # we need to convert keys to symbols first!
       row.keys.each do |k|
-        next if k.is_a? Symbol 
-        row[k.to_sym] = row[k] 
-        row.delete(k) 
+        next if k.is_a? Symbol
+        row[k.to_sym] = row[k]
+        row.delete(k)
       end
-      self.call(row)
+      call(row)
     end
   end
-      
 end