litestack 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1125a2cb5a2d9ea277fb9a0774632eb297ed4d322882dd33776d8a8f1c6471e
-  data.tar.gz: 9cb917c999850b2bd668e8826a4be4f091ee4e8e0f963b54ed87f5b7a7b060d6
+  metadata.gz: 11ef03890b2883bd21fb443d959774f9932f56e927d2ee2a40a028713526ce9c
+  data.tar.gz: c531925eeffa84973475c14d4d230d8919d5e5ec095e5055f9bab6b0f94519e3
 SHA512:
-  metadata.gz: e51460d99e66732e3dcd72e9444f1d739df7ae37804eb9b308cdae0faa7c9c34d4e7554e8f9d0bf91905f7ae3e9b4d7470509a84653da2ee736582b268fe188a
-  data.tar.gz: 873b188bd6db924f1e56034c22f5cc465b01d653ac925a2d30de648271b0b64a705894667b90005bd3d457ad318d3ee4736b16a60fa128c34d894172bc9b8e16
+  metadata.gz: 82c0878fb57fa89290c6550ddbf5b393e436883252285dd78083ba7e96f78947aa3fdf3e92d96f3baec6b9b2b0d2edfdc470f4e1c544427f3da8db704179de27
+  data.tar.gz: 358bc249c3f1371714e12f4df0a0d82883e42eec14528f4faaef8a5e611e62af1039d8c00ebdc9d89be1a514a3bfd698ce3f988fa1b4f29eb1cf72fd1ca03b25

data/BENCHMARKS.md

@@ -3,6 +3,8 @@
 This is a set of initial (simple) benchmars, designed to understand the baseline performance for different litestack components against their counterparts. 
 These are not real life scenarios and I hope I will be able to produce some interesting ones soon.
 
+All these benchmarks were run on an 8 core, 16 thread, AMD 5700U based laptop, in a Virtual Box VM
+
 > ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
 
 ### Point Read
@@ -109,5 +111,14 @@ Two scenarios were benchmarked, an empty job and one with a 100ms sleep to simul
 
 Running Litejob with fibers is producing much faster results than any threaded solution. Still though, threaded Litejob remains ahead of Sidekiq in all scenarios. 
 
+> ![litecable](https://github.com/oldmoe/litestack/blob/master/assets/litecable_logo_teal.png?raw=true)
+
+A client written using the Iodine web server was used to generate the WS load in an event driven fashion. The Rails application, the Iodine based load generator and the Redis server were all run on the same machine to exclude network overheads (Redis still pays for the TCP stack overhead though)
 
+|Requests|Redis Req/Sec|Litestack Req/sec|Redis p90 Latency (ms)|Litestack p90 Latency (ms)|Redis p99 Latency (ms)|Litestack p99 Latancy (ms)| 
+|-:|-:|-:|-:|-:|-:|-:|
+|1,000|2611|3058|34|27|153|78|
+|10,000|3110|5328|81|40|138|122
+|100,000|3403|5385|41|36|153|235
 
+On average, Litecable is quite faster than the Redis based version and offers better latenices for over 90% of the requests, though Redis usually delivers better p99 latencies, 

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.4.1] - 2023-10-11
+
+- Add missing Litesearch::Model dependency
+
+## [0.4.0] - 2023-10-11
+
+- Introduced Litesearch, dynamic & fast full text search capability for Litedb
+- ActiveRecord and Sequel integration for Litesearch
+- Slight improvement to the Sequel Litedb adapter for better Litesearch integration
+
 ## [0.3.0] - 2023-08-13
 
 - Reworked the Litecable thread safety model

data/Gemfile

@@ -8,3 +8,5 @@ gemspec
 
 
 gem "rack", "~> 3.0"
+
+gem "simplecov"

data/bench/bench_jobs_rails.rb

@@ -29,7 +29,7 @@ if env == "a" # threaded
 end
 
 require_relative '../lib/active_job/queue_adapters/litejob_adapter'
-puts Litesupport.scheduler
+puts Litescheduler.backend
 
 RailsJob.queue_adapter = :litejob
 t = Time.now.to_f

data/bench/bench_jobs_raw.rb

@@ -28,7 +28,7 @@ end
 
 require './uljob.rb'
 
-STDERR.puts "litejob started in #{Litesupport.scheduler} environmnet"
+STDERR.puts "litejob started in #{Litescheduler.backend} environmnet"
 
 t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 bench("enqueuing litejobs", count) do |i|

data/bench/uljob.rb

@@ -1,5 +1,5 @@
 require './bench'
-require '../lib/litestack'
+require '../lib/litestack/litejob'
 
 class MyJob
   include Litejob

data/lib/active_support/cache/litecache.rb

@@ -48,7 +48,7 @@ module ActiveSupport
         @cache.prune(limit)
       end 
 
-      def clear()
+      def clear(options = nil)
       	@cache.clear
       end
       

data/lib/litestack/litecable.rb

@@ -80,7 +80,7 @@ class Litecable
   end
 
   def create_broadcaster
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         @messages.acquire do |msgs|
           if msgs.length > 0
@@ -97,7 +97,7 @@ class Litecable
   end
 
   def create_pruner
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         run_stmt(:prune, @options[:expire_after])
         sleep @options[:expire_after]
@@ -106,7 +106,7 @@ class Litecable
   end
 
   def create_listener
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
         run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|

data/lib/litestack/litecache.rb

@@ -212,7 +212,7 @@ class Litecache
   end
     
   def spawn_worker
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running
         @conn.acquire do |cache|
           begin

data/lib/litestack/litedb.rb

@@ -4,12 +4,18 @@ require_relative 'litesupport'
 # all measurable components should require the litemetric class
 require_relative 'litemetric'
 
+# litedb in particular gets access to litesearch
+require_relative 'litesearch'
+
 # Litedb inherits from the SQLite3::Database class and adds a few initialization options
 class Litedb < ::SQLite3::Database
 
   # add litemetric support
   include Litemetric::Measurable
   
+  # add litesearch support
+  include Litesearch
+  
   # overrride the original initilaizer to allow for connection configuration
   def initialize(file, options = {}, zfs = nil )
     if block_given?

data/lib/litestack/litejob.rb

@@ -68,9 +68,8 @@ module Litejob
       get_jobqueue
     end
        
-    def delete(id, queue_name=nil) 
-      queue_name ||= queue
-      get_jobqueue.delete(id, queue)
+    def delete(id) 
+      get_jobqueue.delete(id)
     end  
          
     def queue

data/lib/litestack/litejobqueue.rb

@@ -54,7 +54,7 @@ class Litejobqueue < Litequeue
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
-    @@queue ||= Litesupport.synchronize{self.new(options)}
+    @@queue ||= Litescheduler.synchronize{self.new(options)}
   end
 
   def self.new(options = {})
@@ -95,7 +95,7 @@ class Litejobqueue < Litequeue
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
-    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
+    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue}, mode: :strict)
     res = super(payload, delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[1]} class:#{jobclass} job:#{res[0]}")
@@ -103,7 +103,7 @@ class Litejobqueue < Litequeue
   end
   
   def repush(id, job, delay=0, queue=nil)
-    res = super(id, Oj.dump(job), delay, queue)
+    res = super(id, Oj.dump(job, mode: :strict), delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[0]} class:#{job[:klass]} job:#{id}")
     res
@@ -121,7 +121,7 @@ class Litejobqueue < Litequeue
   def delete(id)
     job = super(id)
     @logger.info("[litejob]:[DEL] job: #{job}")
-    job = Oj.load(job[0]) if job
+    job = Oj.load(job[0], symbol_keys: true) if job
     job
   end
   
@@ -163,17 +163,17 @@ class Litejobqueue < Litequeue
   end
   
   def job_started
-    Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
+    Litescheduler.synchronize(@mutex){@jobs_in_flight += 1}
   end
   
   def job_finished
-    Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
+    Litescheduler.synchronize(@mutex){@jobs_in_flight -= 1}
   end
     
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
-      Litesupport.spawn &block
+      Litescheduler.spawn &block
     else
       yield
     end
@@ -181,50 +181,23 @@ class Litejobqueue < Litequeue
     
   # create a worker according to environment
   def create_worker
-    Litesupport.spawn do
+    Litescheduler.spawn do
       worker_sleep_index = 0
       while @running do
         processed = 0
-        @queues.each do |level| # iterate through the levels
-          level[1].each do |q| # iterate through the queues in the level
-            index = 0
-            max = level[0]
-            while index < max && payload = pop(q[0], 1) # fearlessly use the same queue object 
-              capture(:dequeue, q[0])
+        @queues.each do |priority, queues| # iterate through the levels
+          queues.each do |queue, spawns| # iterate through the queues in the level
+            batched = 0
+            
+            while (batched < priority) && (payload = pop(queue, 1)) # fearlessly use the same queue object
+              capture(:dequeue, queue)
               processed += 1
-              index += 1
-              begin
-                id, job = payload[0], payload[1]
-                job = Oj.load(job)
-                @logger.info "[litejob]:[DEQ] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
-                klass = eval(job[:klass])
-                schedule(q[1]) do # run the job in a new context
-                  job_started #(Litesupport.current_context)
-                  begin
-                    measure(:perform, q[0]){ klass.new.perform(*job[:params]) }
-                    @logger.info "[litejob]:[END] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
-                  rescue Exception => e
-                    # we can retry the failed job now
-                    capture(:fail, q[0])
-                    if job[:retries] == 0
-                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
-                      repush(id, job, @options[:dead_job_retention], '_dead')
-                    else
-                      capture(:retry, q[0])
-                      retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
-                      job[:retries] -=  1
-                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
-                      repush(id, job, retry_delay, q[0])                      
-                    end
-                  end
-                  job_finished #(Litesupport.current_context)
-                end
-              rescue Exception => e
-                # this is an error in the extraction of job info, retrying here will not be useful
-                @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
-                job_finished #(Litesupport.current_context)
-              end
-              Litesupport.switch #give other contexts a chance to run here
+              batched += 1
+              
+              id, serialized_job = payload
+              process_job(queue, id, serialized_job, spawns)
+              
+              Litescheduler.switch # give other contexts a chance to run here
             end
           end
         end
@@ -240,7 +213,7 @@ class Litejobqueue < Litequeue
   
   # create a gc for dead jobs
   def create_garbage_collector
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         while jobs = pop('_dead', 100)
           if jobs[0].is_a? Array
@@ -254,4 +227,34 @@ class Litejobqueue < Litequeue
     end
   end
   
+  def process_job(queue, id, serialized_job, spawns)
+    job = Oj.load(serialized_job)
+    @logger.info "[litejob]:[DEQ] queue:#{queue} class:#{job["klass"]} job:#{id}"
+    klass = Object.const_get(job["klass"])
+    schedule(spawns) do # run the job in a new context
+      job_started # (Litesupport.current_context)
+      begin
+        measure(:perform, queue) { klass.new.perform(*job["params"]) }
+        @logger.info "[litejob]:[END] queue:#{queue} class:#{job["klass"]} job:#{id}"
+      rescue Exception => e # standard:disable Lint/RescueException
+        # we can retry the failed job now
+        capture(:fail, queue)
+        if job["retries"] == 0
+          @logger.error "[litejob]:[ERR] queue:#{queue} class:#{job["klass"]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+          repush(id, job, @options[:dead_job_retention], "_dead")
+        else
+          capture(:retry, queue)
+          retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job["retries"]) * @options[:retry_delay]
+          job["retries"] -= 1
+          @logger.error "[litejob]:[ERR] queue:#{queue} class:#{job["klass"]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
+          repush(id, job, retry_delay, queue)
+        end
+      end
+      job_finished # (Litesupport.current_context)
+    end
+  rescue Exception => e # standard:disable Lint/RescueException
+    # this is an error in the extraction of job info, retrying here will not be useful
+    @logger.error "[litejob]:[ERR] failed to extract job info for: #{serialized_job} with #{e}:#{e.message}"
+    job_finished # (Litesupport.current_context)
+  end
 end

data/lib/litestack/litemetric.rb

@@ -179,7 +179,7 @@ class Litemetric
   end
   
   def create_flusher
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         sleep @options[:flush_interval]
         flush
@@ -188,7 +188,7 @@ class Litemetric
   end
   
   def create_summarizer
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         sleep @options[:summarize_interval]
         summarize
@@ -211,7 +211,7 @@ class Litemetric
     end
 
     def create_snapshotter
-      Litesupport.spawn do
+      Litescheduler.spawn do
         while @running do
           sleep @litemetric.options[:snapshot_interval]
           capture_snapshot

data/lib/litestack/litescheduler.rb

@@ -0,0 +1,84 @@
+# frozen_stringe_literal: true
+
+module Litescheduler
+  # cache the scheduler we are running in
+  # it is an error to change the scheduler for a process
+  # or for a child forked from that process
+  def self.backend
+    @backend ||= if Fiber.scheduler
+      :fiber
+    elsif defined? Polyphony
+      :polyphony
+    elsif defined? Iodine
+      :iodine 
+    else
+      :threaded
+    end
+  end
+  
+  # spawn a new execution context
+  def self.spawn(&block)
+    if backend == :fiber
+      Fiber.schedule(&block)
+    elsif backend == :polyphony
+      spin(&block)
+    elsif backend == :threaded or backend == :iodine
+      Thread.new(&block)
+    end
+    # we should never reach here
+  end
+  
+  def self.storage
+    if backend == :fiber || backend == :poylphony
+      Fiber.current.storage
+    else
+      Thread.current
+    end
+  end
+  
+  def self.current
+    if backend == :fiber || backend == :poylphony
+      Fiber.current
+    else
+      Thread.current
+    end
+  end
+  
+  # switch the execution context to allow others to run
+  def self.switch
+    if backend == :fiber
+      Fiber.scheduler.yield
+      true
+    elsif backend == :polyphony
+      Fiber.current.schedule
+      Thread.current.switch_fiber
+      true
+    else
+      #Thread.pass
+      false
+    end   
+  end
+  
+  # bold assumption, we will only synchronize threaded code!
+  # If some code explicitly wants to synchronize a fiber
+  # they must send (true) as a parameter to this method
+  # else it is a no-op for fibers
+  def self.synchronize(fiber_sync = false, &block)
+    if backend == :fiber or backend == :polyphony
+      yield # do nothing, just run the block as is
+    else
+      self.mutex.synchronize(&block)
+    end
+  end
+  
+  def self.max_contexts
+    return 50 if backend == :fiber || backend == :polyphony
+    5    
+  end
+  
+  # mutex initialization
+  def self.mutex
+    # a single mutex per process (is that ok?)
+    @@mutex ||= Mutex.new
+  end
+end

data/lib/litestack/litesearch/index.rb

@@ -0,0 +1,230 @@
+require 'oj'
+require_relative './schema.rb'
+
+class Litesearch::Index
+  
+  DEFAULT_SEARCH_OPTIONS = {limit: 25, offset: 0}
+
+  def initialize(db, name)
+    @db = db # this index instance will always belong to this db instance
+    @stmts = {}
+    name = name.to_s.downcase.to_sym
+    # if in the db then put in cache and return if no schema is given
+    # if a schema is given then compare the new and the existing schema
+    # if they are the same put in cache and return
+    # if they differ only in weights then set the new weights, update the schema, put in cache and return
+    # if they differ in fields (added/removed/renamed) then update the structure, then rebuild if auto-rebuild is on
+    # if they differ in tokenizer then rebuild if auto-rebuild is on (error otherwise)
+    # if they differ in both then update the structure and rebuild if auto-rebuild is on (error otherwise)
+    load_index(name) if exists?(name)
+    
+    if block_given?
+      schema = Litesearch::Schema.new
+      schema.schema[:name] = name
+      yield schema
+      schema.post_init
+      # now that we have a schema object we need to check if we need to create or modify and existing index
+      @db.transaction(:immediate) do
+        if exists?(name)
+          load_index(name)
+          do_modify(schema)
+        else
+          do_create(schema)
+        end
+        prepare_statements
+      end
+    else
+      if exists?(name)
+        # an index already exists, load it from the database and return the index instance to the caller
+        load_index(name)
+        prepare_statements
+      else
+        raise "index does not exist and no schema was supplied"
+      end
+    end
+  end
+  
+  def load_index(name)
+    # we cannot use get_config_value here since the schema object is not created yet, should we allow something here?
+    @schema = Litesearch::Schema.new(Oj.load(@db.get_first_value("SELECT v from #{name}_config where k = ?", :litesearch_schema.to_s))) rescue nil
+    raise "index configuration not found, either corrupted or not a litesearch index!" if @schema.nil?
+    self
+  end
+          
+  def modify
+    schema = Litesearch::Schema.new
+    yield schema
+    schema.schema[:name] = @schema.schema[:name]
+    do_modify(schema)
+  end
+  
+  def rebuild!
+    @db.transaction(:immediate) do
+      do_rebuild
+    end
+  end
+  
+  def add(document)
+    @stmts[:insert].execute!(document)
+    return @db.last_insert_row_id
+  end
+  
+  def remove(id)
+    @stmts[:delete].execute!(id)
+  end
+  
+  def count(term = nil)
+    if term
+      @stmts[:count].execute!(term)[0][0]
+    else
+      @stmts[:count_all].execute!()[0][0]
+    end
+  end
+  
+  # search options include
+  # limit: how many records to return
+  # offset: start from which record
+  def search(term, options = {})
+    result = []
+    options = DEFAULT_SEARCH_OPTIONS.merge(options)
+    rs = @stmts[:search].execute(term, options[:limit], options[:offset])
+    if @db.results_as_hash
+      rs.each_hash do |hash|
+        result << hash 
+      end
+    else
+      result = rs.to_a
+    end    
+    result
+  end
+      
+  def clear!
+    @stmts[:delete_all].execute!(id)
+  end
+  
+  def drop!
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
+      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+        @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
+      end
+    end
+    @db.execute(@schema.sql_for(:drop))
+  end
+  
+  
+  private
+  
+  def exists?(name)
+    @db.get_first_value("SELECT count(*) FROM SQLITE_MASTER WHERE name = ? AND type = 'table' AND (sql like '%fts5%' OR sql like '%FTS5%')", name.to_s) == 1
+  end
+
+  def prepare_statements
+    stmt_names = [:insert, :delete, :delete_all, :drop, :count, :count_all, :search]
+    stmt_names.each do |stmt_name|
+      @stmts[stmt_name] = @db.prepare(@schema.sql_for(stmt_name))
+    end      
+  end
+  
+  def do_create(schema)   
+    @schema = schema   
+    @schema.clean
+    # create index
+    @db.execute(schema.sql_for(:create_index, true))
+    # adjust ranking function
+    @db.execute(schema.sql_for(:ranks, true))
+    # create triggers (if any)
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:create_primary_triggers))
+      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+        @db.execute_batch(secondary_triggers_sql)
+      end
+      @db.execute(@schema.sql_for(:rebuild)) if @schema.get(:rebuild_on_create)
+    end
+    set_config_value(:litesearch_schema, @schema.schema)
+  end
+
+  def do_modify(new_schema)
+    changes = @schema.compare(new_schema)
+    # ensure the new schema maintains feild order
+    new_schema.order_fields(@schema)
+    # with the changes object decide what needs to be done to the schema
+    requires_schema_change = false
+    requires_trigger_change = false
+    requires_rebuild = false
+    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:filter_column] || changes[:removed_fields_count] > 0# any change here will require a schema change
+      requires_schema_change = true
+      # only a change in tokenizer
+      requires_rebuild = changes[:tokenizer] || new_schema.get(:rebuild_on_modify)
+      requires_trigger_change = (changes[:table] || changes[:fields] || changes[:filter_column]) && @schema.get(:type) == :backed 
+    end
+    if requires_schema_change
+      # 1. enable schema editing
+      @db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
+      # 2. update the index sql
+      @db.execute(new_schema.sql_for(:update_index), new_schema.sql_for(:create_index))
+      # 3. update the content table sql (if it exists)
+      @db.execute(new_schema.sql_for(:update_content_table), new_schema.sql_for(:create_content_table, new_schema.schema[:fields].count))
+      # adjust shadow tables
+      @db.execute(new_schema.sql_for(:expand_data), changes[:extra_fields_count])
+      @db.execute(new_schema.sql_for(:expand_docsize), changes[:extra_fields_count])
+      @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
+      # need to reprepare statements        
+    end
+    if requires_trigger_change
+      @db.execute_batch(new_schema.sql_for(:drop_primary_triggers))
+      @db.execute_batch(new_schema.sql_for(:create_primary_triggers))
+      if secondary_triggers_sql = new_schema.sql_for(:create_secondary_triggers)
+        @db.execute_batch(new_schema.sql_for(:drop_secondary_triggers))
+        @db.execute_batch(secondary_triggers_sql)
+      end
+    end
+    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:weights] || changes[:filter_column]
+      @schema = new_schema
+      set_config_value(:litesearch_schema, @schema.schema)
+      prepare_statements
+      #save_schema
+    end
+    do_rebuild if requires_rebuild
+    # update the weights if they changed 
+    @db.execute(@schema.sql_for(:ranks)) if changes[:weights]
+  end
+
+  def do_rebuild
+    # remove any zero weight columns
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
+      if secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers)
+        @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
+      end
+      @db.execute(@schema.sql_for(:drop))
+      @db.execute(@schema.sql_for(:create_index, true))
+      @db.execute_batch(@schema.sql_for(:create_primary_triggers))
+      @db.execute_batch(secondary_triggers_sql) if secondary_triggers_sql
+      @db.execute(@schema.sql_for(:rebuild)) 
+    elsif @schema.get(:type) == :standalone
+      removables = []
+      @schema.get(:fields).each_with_index{|f, i| removables << [f[0], i] if f[1][:weight] == 0  }
+      removables.each do |col|
+        @db.execute(@schema.sql_for(:drop_content_col, col[1]))
+        @schema.get(:fields).delete(col[0]) 
+      end
+      @db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
+      @db.execute(@schema.sql_for(:update_index), @schema.sql_for(:create_index, true))
+      @db.execute(@schema.sql_for(:update_content_table), @schema.sql_for(:create_content_table, @schema.schema[:fields].count)) 
+      @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
+      @db.execute(@schema.sql_for(:rebuild)) 
+    end
+    set_config_value(:litesearch_schema, @schema.schema)
+    @db.execute(@schema.sql_for(:ranks, true)) 
+ end
+  
+  def get_config_value(key)
+    Oj.load(@db.get_first_value(@schema.sql_for(:get_config_value), key.to_s)) #rescue nil
+  end
+  
+  def set_config_value(key, value)
+    @db.execute(@schema.sql_for(:set_config_value), key.to_s, Oj.dump(value))
+  end
+
+end