litestack 0.3.0 → 0.4.1

data/lib/litestack/litesearch/model.rb

@@ -0,0 +1,178 @@
+module Litesearch::Model
+                      
+  def self.included(klass)
+    klass.include InstanceMethods
+    klass.extend ClassMethods 
+    klass.attribute :search_rank, :float if klass.respond_to? :attribute
+    if defined?(Sequel::Model) != nil && klass.ancestors.include?(Sequel::Model)
+       klass.include Litesearch::Model::SequelInstanceMethods
+       klass.extend Litesearch::Model::SequelClassMethods
+       Sequel::Model.extend Litesearch::Model::BaseClassMethods
+    elsif defined?(ActiveRecord::Base) != nil && klass.ancestors.include?(ActiveRecord::Base)
+       klass.include Litesearch::Model::ActiveRecordInstanceMethods
+       klass.extend Litesearch::Model::ActiveRecordClassMethods
+       ActiveRecord::Base.extend Litesearch::Model::BaseClassMethods
+    end
+  end
+  
+  module BaseClassMethods
+    def search_models
+      @@models ||= {}
+    end 
+  end
+  
+  module InstanceMethods
+  
+  end
+  
+  module ClassMethods  
+     
+    def litesearch
+      idx = get_connection.search_index(index_name) do |schema|
+        schema.type :backed
+        schema.table table_name.to_sym
+        yield schema
+        schema.post_init
+        @schema = schema #save the schema   
+      end
+      if defined?(Sequel::Model) != nil && self.ancestors.include?(Sequel::Model)
+        Sequel::Model.search_models[self.name] = self
+      elsif defined?(ActiveRecord::Base) != nil && self.ancestors.include?(ActiveRecord::Base)
+        ActiveRecord::Base.search_models[self.name] = self
+      end
+      idx
+    end
+    
+    def rebuild_index!
+      get_connection.search_index(index_name).rebuild!
+    end
+    
+    def drop_index!
+      get_connection.search_index(index_name).drop!
+    end
+    
+    def search_all(term, options={})
+      options[:offset] ||= 0
+      options[:limit] ||= 25
+      selects = [] 
+      if models = options[:models]
+        models_hash = {}
+        models.each do |model|
+          models_hash[model.name] = model
+        end
+      else
+        models_hash = search_models
+      end
+      models_hash.each do |name, klass|
+        selects << "SELECT '#{name}' AS model, rowid, -rank AS search_rank FROM #{index_name_for_table(klass.table_name)}('#{term}')"
+      end
+      conn = get_connection
+      sql = selects.join(" UNION ") << " ORDER BY search_rank DESC LIMIT #{options[:limit]} OFFSET #{options[:offset]}"
+      result = []
+      rs = conn.query(sql) #, options[:limit], options[:offset])
+      rs.each_hash do |row|
+        obj = models_hash[row["model"]].fetch_row(row["rowid"])
+        obj.search_rank = row["search_rank"]
+        result << obj
+      end
+      rs.close
+      result
+    end
+          
+    # AR specific
+                     
+    private
+    
+    def index_name
+      "#{table_name}_search_idx"
+    end
+    
+    def index_name_for_table(table)
+      "#{table}_search_idx"
+    end
+    
+    # create a new instance of self with the row as an argument
+    def create_instance(row)
+      self.new(row)
+    end
+    
+          
+  end
+  
+  module ActiveRecordInstanceMethods;end
+    
+  module ActiveRecordClassMethods
+    
+    def get_connection
+      connection.raw_connection
+    end
+    
+    def fetch_row(id)
+      find(id)
+    end
+    
+    def search(term)
+      self.select(
+        "#{table_name}.*"
+      ).joins(
+        "INNER JOIN #{index_name} ON #{table_name}.id = #{index_name}.rowid AND rank != 0 AND #{index_name} MATCH ", Arel.sql("'#{term}'")
+      ).select(
+        "-#{index_name}.rank AS search_rank"
+      ).order(
+        Arel.sql("#{index_name}.rank")
+      )        
+    end
+
+    private
+    
+    def create_instance(row)
+      instantiate(row)
+    end
+  end
+  
+  module SequelInstanceMethods
+  
+    def search_rank
+      @values[:search_rank]
+    end
+    
+    def search_rank=(rank)
+      @values[:search_rank] = rank
+    end
+        
+  end
+  
+  module SequelClassMethods
+        
+    def fetch_row(id)
+      self[id]
+    end
+
+    def get_connection
+      db.instance_variable_get(:@raw_db)
+    end
+
+    def search(term)
+      dataset.select(
+        Sequel.lit("#{table_name}.*, -#{index_name}.rank AS search_rank")
+      ).inner_join(
+        Sequel.lit("#{index_name}('#{term}') ON #{table_name}.id = #{index_name}.rowid AND rank != 0")
+      ).order(
+        Sequel.lit('rank')
+      )
+    end
+
+    private
+    
+    def create_instance(row)
+      # we need to convert keys to symbols first!
+      row.keys.each do |k|
+        next if k.is_a? Symbol 
+        row[k.to_sym] = row[k] 
+        row.delete(k) 
+      end
+      self.call(row)
+    end
+  end
+      
+end

data/lib/litestack/litesearch/schema.rb

@@ -0,0 +1,193 @@
+require_relative './schema_adapters.rb'
+
+class Litesearch::Schema
+
+  TOKENIZERS = { 
+    porter: 'porter unicode61 remove_diacritics 2', 
+    unicode: 'unicode61 remove_diacritics 2', 
+    ascii: 'ascii', 
+    trigram: 'trigram' 
+  }
+  
+  INDEX_TYPES = {
+    standalone: Litesearch::Schema::StandaloneAdapter, 
+    contentless: Litesearch::Schema::ContentlessAdapter, 
+    backed: Litesearch::Schema::BackedAdapter
+  }
+
+  DEFAULT_SCHEMA = {
+    name: nil,
+    type: :standalone,
+    fields: nil, 
+    table: nil,
+    filter_column: nil,
+    tokenizer: :porter,
+    auto_create: true,
+    auto_modify: true,
+    rebuild_on_create: false,      
+    rebuild_on_modify: false      
+  }
+
+  attr_accessor :schema
+
+  def initialize(schema = {})
+    @schema = schema #DEFAULT_SCHEMA.merge(schema)
+    @schema[:fields] = {} unless @schema[:fields]
+  end
+
+  # schema definition API
+  def name(new_name)
+    @schema[:name] = new_name
+  end
+    
+  def type(new_type)
+    raise "Unknown index type" if INDEX_TYPES[new_type].nil?
+    @schema[:type] = new_type
+  end
+
+  def table(table_name)
+    @schema[:table] = table_name
+  end
+
+  def fields(field_names)
+    field_names.each {|f| field f }
+  end
+  
+  def field(name, attributes = {})
+    name = name.to_s.downcase.to_sym
+    attributes = {weight: 1}.merge(attributes).select{|k, v| allowed_attributes.include?(k)} # only allow attributes we know, to ease schema comparison later
+    @schema[:fields][name] = attributes
+  end
+  
+  def tokenizer(new_tokenizer)
+    raise "Unknown tokenizer" if TOKENIZERS[new_tokenizer].nil?
+    @schema[:tokenizer] = new_tokenizer    
+  end
+  
+  def filter_column(filter_column)
+    @schema[:filter_column] = filter_column
+  end
+  
+  def auto_create(boolean)
+    @schema[:auto_create] = boolean
+  end
+  
+  def auto_modify(boolean)
+    @schema[:auto_modify] = boolean
+  end
+  
+  def rebuild_on_create(boolean)
+    @schema[:rebuild_on_create] = boolean  
+  end
+   
+  def rebuild_on_modify(boolean)
+    @schema[:rebuild_on_modify] = boolean  
+  end
+
+  def post_init
+    @schema = DEFAULT_SCHEMA.merge(@schema)
+  end
+
+  # schema sql generation API
+  
+  def sql_for(method, *args)
+    adapter.sql_for(method, *args)
+  end
+    
+  # schema data structure API
+  def get(key)
+    @schema[key]
+  end  
+  
+  def get_field(name)
+    @schema[:fields][name]
+  end
+  
+  def adapter
+    @adapter ||= INDEX_TYPES[@schema[:type]].new(@schema)
+  end
+  
+  def reset_sql
+    adapter.generate_sql
+  end
+  
+  def order_fields(old_schema)
+    adapter.order_fields(old_schema)
+  end
+  
+  # should we do this at the schema objects level?
+  def compare(other_schema)
+    other_schema = other_schema.schema
+    # are the schemas identical?
+    # 1 - same fields?
+    [:type, :tokenizer, :name, :table].each do |key|
+       other_schema[key] = @schema[key] if other_schema[key].nil?  
+    end
+    if @schema[:type] != other_schema[:type]
+      raise Litesearch::SchemaChangeException.new "Cannot change the index type, please drop the index before creating it again with the new type"
+    end
+    changes = { tokenizer: @schema[:tokenizer] != other_schema[:tokenizer], table: @schema[:table] != other_schema[:table], removed_fields_count: 0, filter_column: @schema[:filter_column] != other_schema[:filter_column]  }
+    #check tokenizer changes
+    if changes[:tokenizer] && !other_schema[:rebuild_on_modify]
+      raise Litesearch::SchemaChangeException.new "Cannot change the tokenizer without an index rebuild!"
+    end
+
+    
+
+    # check field changes
+    keys = @schema[:fields].keys.sort
+    other_keys = other_schema[:fields].keys.sort
+
+    extra_keys = other_keys - keys
+    extra_keys.each do |key|
+      if other_schema[:fields][key][:weight] == 0
+        other_schema[:fields].delete(key)
+      end
+    end
+    
+    other_keys = other_schema[:fields].keys.sort
+
+    changes[:fields] = keys != other_keys # only acceptable change is adding extra fields
+    changes[:extra_fields_count] = other_keys.count - keys.count 
+    # check for missing fields (please note that adding fields can work without a rebuild)
+    if keys - other_keys != []
+      raise Litesearch::SchemaChangeException.new "Missing fields from existing schema, they have to exist with weight zero until the next rebuild!" 
+    end
+    
+    # check field weights
+    weights = keys.collect{|key| @schema[:fields][key][:weight] }
+    other_weights = other_keys.collect{|key| other_schema[:fields][key][:weight] }
+    changes[:weights] = weights != other_weights # will always be true if fields are added
+    if (removed_count = other_weights.select{|w| w == 0}.count) > 0
+      changes[:removed_fields_count] = removed_count
+    end
+    # check field attributes, only backed tables have attributes
+    attrs = keys.collect do |key| 
+      f = @schema[:fields][key].dup
+      f.delete(:weight)
+      f.select{|k,v| allowed_attributes.include? k }
+    end
+    other_attrs = other_keys.collect do |key|
+      f = other_schema[:fields][key].dup
+      f.delete(:weight)
+      f.select{|k,v| allowed_attributes.include? k }
+    end
+    changes[:attributes] if other_attrs != attrs # this means that we will need to redefine the triggers if any are there and also the table definition if needed
+    
+    # return the changes
+    changes
+  end
+  
+  def clean
+    removable = @schema[:fields].select{|name, f| f[:weight] == 0 }.collect{|name, f| name}
+    removable.each{|name| @schema[:fields].delete(name)}
+  end
+  
+  def allowed_attributes
+    [:weight, :col, :target]
+  end
+    
+end
+
+class Litesearch::SchemaException < StandardError; end
+class Litesearch::SchemaChangeException < StandardError; end

data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb

@@ -0,0 +1,147 @@
+class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
+  
+  private
+
+  def table
+    @schema[:table]
+  end
+
+  def generate_sql
+    super
+    @sql[:rebuild] = :rebuild_sql
+    @sql[:drop_primary_triggers] = :drop_primary_triggers_sql
+    @sql[:drop_secondary_triggers] = :drop_secondary_triggers_sql
+    @sql[:create_primary_triggers] = :create_primary_triggers_sql
+    @sql[:create_secondary_triggers] = :create_secondary_triggers_sql
+  end
+  
+  def drop_primary_triggers_sql
+sql = <<-SQL 
+DROP TRIGGER IF EXISTS #{name}_insert;
+DROP TRIGGER IF EXISTS #{name}_update;
+DROP TRIGGER IF EXISTS #{name}_update_not;
+DROP TRIGGER IF EXISTS #{name}_delete;
+SQL
+  end
+  
+  def create_primary_triggers_sql(active=false)
+    when_stmt = "TRUE"
+    cols = active_cols_names
+    if filter = @schema[:filter_column]
+      when_stmt = "NEW.#{filter} = TRUE"
+      cols << filter
+    end
+
+sql = <<-SQL
+  CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
+    INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
+  END;
+  CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN #{when_stmt} BEGIN
+    INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
+  END;
+  CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN NOT #{when_stmt} BEGIN
+    DELETE FROM #{name} WHERE rowid = NEW.rowid;
+  END;
+  CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
+    DELETE FROM #{name} WHERE rowid = OLD.id;
+  END;
+SQL
+  end
+  
+  def drop_secondary_trigger_sql(target_table, target_col, col)
+    "DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
+  end
+  
+  def create_secondary_trigger_sql(target_table, target_col, col)
+sql = <<-SQL
+CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
+  #{rebuild_sql} AND #{table}.#{col} = NEW.id;
+END;
+SQL
+  end
+
+  def drop_secondary_triggers_sql
+    sql = ""
+    @schema[:fields].each do |name, field|
+      if field[:trigger_sql]
+        sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+      end
+    end
+    return sql.empty? ? nil : sql
+  end
+
+  
+  def create_secondary_triggers_sql
+    sql = ""  
+    @schema[:fields].each do |name, field|
+      if field[:trigger_sql]
+        sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+      end
+    end
+    return sql.empty? ? nil : sql
+  end
+  
+  
+  def rebuild_sql
+    conditions = "" 
+    jcs = join_conditions_sql
+    fs = filter_sql
+    conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
+    "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}" 
+  end
+    
+  def enrich_schema
+    @schema[:fields].each do |name, field|      
+      if field[:target] && ! field[:target].start_with?("#{table}.")  
+        field[:target] = field[:target].downcase
+        target_table, target_col = field[:target].split('.') 
+        field[:col] = "#{name}_id".to_sym unless field[:col]
+        field[:target_table] = target_table.to_sym 
+        field[:target_col] = target_col.to_sym
+        field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
+        field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+        field[:target_table_alias] = "#{field[:target_table]}_#{name}"
+      else
+        field[:col] = name unless field[:col]
+        field[:sql] = field[:col]
+        field[:target_table] = @schema[:table]
+        field[:target] = "#{@schema[:table]}.#{field[:sql]}"
+      end
+    end
+  end
+  
+  def filter_sql
+    sql = "" 
+    sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
+    sql
+  end
+
+  def trigger_cols_sql
+    active_fields.collect do |name, field|
+      field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
+    end.join(", ")
+  end
+  
+  def select_cols_sql
+    active_fields.collect do |name, field|
+      field[:trigger_sql] != nil ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
+    end.join(', ')
+  end
+  
+  def join_tables_sql
+    tables = [@schema[:table]]    
+    active_fields.each do |name, field|
+      tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]    
+    end
+    tables.uniq.join(", ")
+  end
+  
+  def join_conditions_sql
+    conditions = []
+    active_fields.each do |name, field|
+      conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]    
+    end
+    conditions.join(" AND ")    
+  end 
+  
+end

data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb

@@ -0,0 +1,128 @@
+class Litesearch::Schema::BasicAdapter
+  
+  def initialize(schema)
+    @schema = schema
+    @sql = {}
+    enrich_schema
+    generate_sql
+  end
+  
+  def name
+    @schema[:name]
+  end
+  
+  def table
+    @schema[:table]
+  end
+
+  def fields
+    @schema[:fields]
+  end
+
+  def field_names
+    @schema[:fields].keys
+  end
+  
+  def active_fields
+    @schema[:fields].select{|k, v| v[:weight] != 0 }
+  end
+  
+  def active_field_names
+    active_fields.keys
+  end
+  
+  def active_cols_names
+    active_fields.collect{|k, v| v[:col]}
+  end
+  
+  def weights
+    @schema[:fields].values.collect{|v| v[:weight].to_f }
+  end
+
+  def active_weights
+    active_fields.values.collect{|v| v[:weight].to_f }
+  end
+
+  def tokenizer_sql
+    Litesearch::Schema::TOKENIZERS[@schema[:tokenizer]]
+  end
+
+  def order_fields(old_schema)
+    new_fields = {}
+    old_field_names = old_schema.schema[:fields].keys
+    old_field_names.each do |name|
+      new_fields[name] = @schema[:fields].delete(name) 
+    end
+    missing_field_names = field_names - old_field_names
+    missing_field_names.each do |name|
+      new_fields[name] = @schema[:fields].delete(name)
+    end   
+    @schema[:fields] = new_fields # this should be in order now
+    generate_sql
+    enrich_schema
+  end
+
+  def sql_for(method, *args)
+    if sql = @sql[method]
+      if sql.is_a? String
+        return sql
+      elsif sql.is_a? Proc
+        return sql.call(*args)
+      elsif sql.is_a? Symbol
+        return self.send(sql, *args)
+      elsif sql.is_a? Litesearch::SchemaChangeException
+        raise sql
+      end
+    end
+  end
+    
+  def generate_sql
+    @sql[:create_index] = :create_index_sql
+    @sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
+    @sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
+    @sql[:count] = "SELECT count(*) FROM #{name}(:term)"
+    @sql[:count_all] = "SELECT count(*) FROM #{name}"
+    @sql[:delete_all] = "DELETE FROM #{name}"
+    @sql[:drop] = "DROP TABLE #{name}"
+    @sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
+    @sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
+    @sql[:ranks] = :ranks_sql 
+    @sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
+    @sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
+    @sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
+    @sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
+    @sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
+  end
+  
+  private
+  
+  def ranks_sql(active=false)
+    if active
+      weights_sql = weights.join(', ')
+    else
+      weights_sql = active_weights.join(', ')      
+    end
+    "INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
+  end
+  
+  def active_col_names_sql
+    active_field_names.join(', ')
+  end
+  
+  def active_col_names_var_sql
+    ":#{active_field_names.join(', :')}"
+  end
+    
+  def col_names_sql
+    field_names.join(', ')
+  end
+  
+  def col_names_var_sql
+    ":#{field_names.join(', :')}"
+  end
+
+  def enrich_schema
+  end
+    
+end
+

data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb

@@ -0,0 +1,17 @@
+class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
+
+  private
+
+  def generate_sql
+    super
+    #@sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
+    #@sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
+  end
+  
+  def create_index_sql(active = false)
+    col_names = active ? active_col_names_sql : col_names_sql
+    "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
+  end
+
+end
+

data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb

@@ -0,0 +1,33 @@
+class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
+  
+  def generate_sql
+    super
+    @sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
+    @sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
+    @sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
+    @sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
+    @sql[:drop_content_table] = "DROP TABLE #{name}_content"
+    @sql[:drop_content_col] = :drop_content_col_sql
+    @sql[:create_content_table] = :create_content_table_sql 
+    @sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
+  end
+
+  private
+
+  def create_index_sql(active = false)
+    col_names = active ? active_col_names_sql : col_names_sql
+    "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
+  end
+    
+  def drop_content_col_sql(col_index)
+    "ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
+  end
+  
+  def create_content_table_sql(count)
+    cols = [] 
+    count.times{|i| cols << "c#{i}" }
+    "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(', ')})"
+  end
+  
+end
+

data/lib/litestack/litesearch/schema_adapters.rb

@@ -0,0 +1,9 @@
+require_relative './schema_adapters/basic_adapter.rb'
+require_relative './schema_adapters/standalone_adapter.rb'
+require_relative './schema_adapters/contentless_adapter.rb'
+require_relative './schema_adapters/backed_adapter.rb'
+
+
+
+
+