litestack 0.3.0 → 0.4.2

data/lib/litestack/litemetric_collector.sql.yml

@@ -1,56 +1,53 @@
 schema:
   1:
-    
     create_events: >
       CREATE TABLE IF NOT EXISTS local_events(
         topic TEXT NOT NULL,
-        name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
-        key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
-        count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
-        value REAL, 
-        minimum REAL, 
-        maximum REAL, 
-        created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE, 
-        resolution TEXT DEFAULT('minute') NOT NULL, 
+        name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
+        key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
+        count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE,
+        value REAL,
+        minimum REAL,
+        maximum REAL,
+        created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE,
+        resolution TEXT DEFAULT('minute') NOT NULL,
         PRIMARY KEY(resolution, created_at, topic, name, key)
-      ) STRICT;        
+      ) STRICT;
 
 stmts:
-      
   capture_event: >
-    INSERT INTO local_events(topic, name, key, created_at, count, value, minimum, maximum) 
-    VALUES 
-      (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?6), 
+    INSERT INTO local_events(topic, name, key, created_at, count, value, minimum, maximum)
+    VALUES
+      (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?6),
       (?1, ?2, '___', ?4, ?5, ?6, ?6, ?6),
-      (?1, '___', '___', ?4, ?5, ?6, ?6, ?6) 
-    ON CONFLICT DO UPDATE 
-    SET 
-      count = count + EXCLUDED.count, 
-      value = value + EXCLUDED.value, 
-      minimum = min(minimum, EXCLUDED.minimum), 
-      maximum = max(maximum, EXCLUDED.maximum) 
-    
-  migrate_events: 
-    INSERT INTO m.events(topic, name, key, created_at, count, value, minimum, maximum) 
-    SELECT topic, name, key, created_at, count, value, minimum, maximum 
-    FROM local_events 
-    ORDER BY resolution, created_at ASC, topic, name, key  
+      (?1, '___', '___', ?4, ?5, ?6, ?6, ?6)
+    ON CONFLICT DO UPDATE
+    SET
+      count = count + EXCLUDED.count,
+      value = value + EXCLUDED.value,
+      minimum = min(minimum, EXCLUDED.minimum),
+      maximum = max(maximum, EXCLUDED.maximum);
+
+  migrate_events: >
+    INSERT INTO m.events(topic, name, key, created_at, count, value, minimum, maximum)
+    SELECT topic, name, key, created_at, count, value, minimum, maximum
+    FROM local_events
+    ORDER BY resolution, created_at ASC, topic, name, key
     LIMIT ?
-    ON CONFLICT DO UPDATE 
-    SET 
-      count = count + EXCLUDED.count, 
-      value = value + EXCLUDED.value, 
-      minimum = min(minimum, EXCLUDED.minimum), 
-      maximum = max(maximum, EXCLUDED.maximum) 
+    ON CONFLICT DO UPDATE
+    SET
+      count = count + EXCLUDED.count,
+      value = value + EXCLUDED.value,
+      minimum = min(minimum, EXCLUDED.minimum),
+      maximum = max(maximum, EXCLUDED.maximum);
 
-  delete_migrated_events:
+  delete_migrated_events: >
     DELETE FROM local_events WHERE rowid IN (
-      SELECT rowid 
-      FROM local_events 
-      ORDER BY resolution, created_at ASC, topic, name, key 
+      SELECT rowid
+      FROM local_events
+      ORDER BY resolution, created_at ASC, topic, name, key
       LIMIT ?
-    )
-  
-  event_count:
+    );
+
+  event_count: >
     SELECT count(*) FROM local_events;
-   

data/lib/litestack/litequeue.rb

@@ -1,27 +1,26 @@
 # frozen_stringe_literal: true
 
 # all components should require the support module
-require_relative 'litesupport'
+require_relative "litesupport"
 
-#require 'securerandom'
+# require 'securerandom'
 
 ##
-#Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
+# Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
 #
-#Litequeue is built on top of SQLite, which makes it very fast and efficient, even when handling large volumes of data. This lightweight and easy-to-use queueing system serves as a good foundation for building more advanced job processing frameworks that require basic queuing capabilities.
+# Litequeue is built on top of SQLite, which makes it very fast and efficient, even when handling large volumes of data. This lightweight and easy-to-use queueing system serves as a good foundation for building more advanced job processing frameworks that require basic queuing capabilities.
 #
-  
-class Litequeue
 
+class Litequeue
   # the default options for the queue
-  # can be overriden by passing new options in a hash 
+  # can be overriden by passing new options in a hash
   # to Litequeue.new
   #   path: "./queue.db"
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
   #   sync: 1 -> sync only when checkpointing
-  
+
   include Litesupport::Liteconnection
-  
+
   DEFAULT_OPTIONS = {
     path: Litesupport.root.join("queue.sqlite3"),
     mmap_size: 32 * 1024,
@@ -35,70 +34,70 @@ class Litequeue
   #   queue.pop # => nil
   #   sleep 2
   #   queue.pop # => "somevalue"
-  
+
   def initialize(options = {})
     init(options)
   end
-   
+
   # push an item to the queue, optionally specifying the queue name (defaults to default) and after how many seconds it should be ready to pop (defaults to zero)
   # a unique job id is returned from this method, can be used later to delete it before it fires. You can push string, integer, float, true, false or nil values
   #
-  def push(value, delay=0, queue='default')
+  def push(value, delay = 0, queue = "default")
     # @todo - check if queue is busy, back off if it is
     # also bring back the synchronize block, to prevent
     # a race condition if a thread hits the busy handler
     # before the current thread proceeds after a backoff
-    #id = SecureRandom.uuid # this is somehow expensive, can we improve?
+    # id = SecureRandom.uuid # this is somehow expensive, can we improve?
     run_stmt(:push, queue, delay, value)[0]
   end
-  
-  def repush(id, value, delay=0, queue='default')
+
+  def repush(id, value, delay = 0, queue = "default")
     run_stmt(:repush, id, queue, delay, value)[0]
   end
-  
-  alias_method :"<<", :push
+
+  alias_method :<<, :push
   alias_method :"<<<", :repush
-  
+
   # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
-  def pop(queue='default', limit = 1)
-   res = run_stmt(:pop, queue, limit)
-   return res[0] if res.length == 1
-   return nil if res.empty?
-   res
+  def pop(queue = "default", limit = 1)
+    res = run_stmt(:pop, queue, limit)
+    return res[0] if res.length == 1
+    return nil if res.empty?
+    res
   end
-  
+
   # delete an item from the queue
   #   queue = Litequeue.new
   #   id = queue.push("somevalue")
   #   queue.delete(id) # => "somevalue"
   #   queue.pop # => nil
   def delete(id)
-    result = run_stmt(:delete, id)[0]  
+    run_stmt(:delete, id)[0]
   end
-  
+
   # deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
-  def clear(queue=nil)
+  def clear(queue = nil)
     run_sql("DELETE FROM queue WHERE iif(?1 IS NOT NULL, name = ?1,  TRUE)", queue)
   end
 
   # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
-  def count(queue=nil)
+  def count(queue = nil)
     run_sql("SELECT count(*) FROM queue WHERE iif(?1 IS NOT NULL, name = ?1, TRUE)", queue)[0][0]
   end
-  
+
   # return the size of the queue file on disk
-  #def size
-  #  run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0] 
-  #end
-  
+  # def size
+  #  run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0]
+  # end
+
   def queues_info
     run_stmt(:info)
   end
-  
+
   def snapshot
     queues = {}
     queues_info.each do |qc|
-      #queues[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
+      # queues[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
       queues[qc[0]] = qc[1]
     end
     {
@@ -109,15 +108,15 @@ class Litequeue
         size: size,
         jobs: count
       },
-      queues: queues      
+      queues: queues
     }
   end
 
-  private  
-      
+  private
+
   def create_connection
     super("#{__dir__}/litequeue.sql.yml") do |conn|
-      conn.wal_autocheckpoint = 10000 
+      conn.wal_autocheckpoint = 10000
       # check if there is an old database and convert entries to the new format
       if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
         conn.transaction(:immediate) do
@@ -127,5 +126,4 @@ class Litequeue
       end
     end
   end
-
-end  
+end

data/lib/litestack/litequeue.sql.yml

@@ -1,45 +1,53 @@
 schema:
-  1:    
-    create_table_queue: >  
+  1:
+    create_table_queue: >
       CREATE TABLE IF NOT EXISTS queue(
-        id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE, 
-        name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE, 
+        id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE,
+        name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE,
         fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE,
         value TEXT,
         created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE
-      ) WITHOUT ROWID 
-      
+      ) WITHOUT ROWID;
+
     create_index_queue_by_name: >
-      CREATE INDEX IF NOT EXISTS idx_queue_by_name ON queue(name, fire_at ASC)
+      CREATE INDEX IF NOT EXISTS idx_queue_by_name ON queue(name, fire_at ASC);
 
 stmts:
 
-  push: INSERT INTO queue(id, name, fire_at, value) VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3) RETURNING id, name
-  
-  repush: INSERT INTO queue(id, name, fire_at, value) VALUES (?, ?, (unixepoch() + ?), ?) RETURNING name 
+  push: >
+    INSERT INTO queue(id, name, fire_at, value)
+    VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3)
+    RETURNING id, name;
+
+  repush: >
+    INSERT INTO queue(id, name, fire_at, value)
+    VALUES (?, ?, (unixepoch() + ?), ?)
+    RETURNING name;
 
   pop: >
-    DELETE FROM queue 
-    WHERE (name, fire_at, id) 
+    DELETE FROM queue
+    WHERE (name, fire_at, id)
     IN (
-        SELECT name, fire_at, id FROM queue 
-        WHERE name = ifnull($1, 'default') 
-        AND fire_at <= (unixepoch()) 
-        ORDER BY fire_at ASC 
-        LIMIT ifnull($2, 1)
-    ) 
-    RETURNING id, value
-  
-  delete: DELETE FROM queue WHERE id = $1 RETURNING value
-  
+      SELECT name, fire_at, id FROM queue
+      WHERE name = ifnull($1, 'default')
+        AND fire_at <= (unixepoch())
+      ORDER BY fire_at ASC
+      LIMIT ifnull($2, 1)
+    )
+    RETURNING id, value;
+
+  delete: >
+    DELETE FROM queue
+    WHERE id = $1
+    RETURNING value;
+
   info: >
-    SELECT 
-      name, 
-      count(*) AS count, 
-      avg(unixepoch() - created_at) AS avg, 
-      min(unixepoch() - created_at) AS min, 
+    SELECT
+      name,
+      count(*) AS count,
+      avg(unixepoch() - created_at) AS avg,
+      min(unixepoch() - created_at) AS min,
       max(unixepoch() - created_at) AS max
-    FROM queue 
-    GROUP BY name 
-    ORDER BY count DESC
-
+    FROM queue
+    GROUP BY name
+    ORDER BY count DESC;

data/lib/litestack/litescheduler.rb

@@ -0,0 +1,84 @@
+# frozen_stringe_literal: true
+
+module Litescheduler
+  # cache the scheduler we are running in
+  # it is an error to change the scheduler for a process
+  # or for a child forked from that process
+  def self.backend
+    @backend ||= if Fiber.scheduler
+      :fiber
+    elsif defined? Polyphony
+      :polyphony
+    elsif defined? Iodine
+      :iodine
+    else
+      :threaded
+    end
+  end
+
+  # spawn a new execution context
+  def self.spawn(&block)
+    if backend == :fiber
+      Fiber.schedule(&block)
+    elsif backend == :polyphony
+      spin(&block)
+    elsif (backend == :threaded) || (backend == :iodine)
+      Thread.new(&block)
+    end
+    # we should never reach here
+  end
+
+  def self.storage
+    if backend == :fiber || backend == :poylphony
+      Fiber.current.storage
+    else
+      Thread.current
+    end
+  end
+
+  def self.current
+    if backend == :fiber || backend == :poylphony
+      Fiber.current
+    else
+      Thread.current
+    end
+  end
+
+  # switch the execution context to allow others to run
+  def self.switch
+    if backend == :fiber
+      Fiber.scheduler.yield
+      true
+    elsif backend == :polyphony
+      Fiber.current.schedule
+      Thread.current.switch_fiber
+      true
+    else
+      # Thread.pass
+      false
+    end
+  end
+
+  # bold assumption, we will only synchronize threaded code!
+  # If some code explicitly wants to synchronize a fiber
+  # they must send (true) as a parameter to this method
+  # else it is a no-op for fibers
+  def self.synchronize(fiber_sync = false, &block)
+    if (backend == :fiber) || (backend == :polyphony)
+      yield # do nothing, just run the block as is
+    else
+      mutex.synchronize(&block)
+    end
+  end
+
+  def self.max_contexts
+    return 50 if backend == :fiber || backend == :polyphony
+    5
+  end
+
+  # mutex initialization
+  def self.mutex
+    # a single mutex per process (is that ok?)
+    @@mutex ||= Mutex.new
+  end
+end

data/lib/litestack/litesearch/index.rb

@@ -0,0 +1,260 @@
+require "oj"
+require_relative "./schema"
+
+class Litesearch::Index
+  DEFAULT_SEARCH_OPTIONS = {limit: 25, offset: 0}
+
+  def initialize(db, name)
+    @db = db # this index instance will always belong to this db instance
+    @stmts = {}
+    name = name.to_s.downcase.to_sym
+    # if in the db then put in cache and return if no schema is given
+    # if a schema is given then compare the new and the existing schema
+    # if they are the same put in cache and return
+    # if they differ only in weights then set the new weights, update the schema, put in cache and return
+    # if they differ in fields (added/removed/renamed) then update the structure, then rebuild if auto-rebuild is on
+    # if they differ in tokenizer then rebuild if auto-rebuild is on (error otherwise)
+    # if they differ in both then update the structure and rebuild if auto-rebuild is on (error otherwise)
+    load_index(name) if exists?(name)
+
+    if block_given?
+      schema = Litesearch::Schema.new
+      schema.schema[:name] = name
+      yield schema
+      schema.post_init
+      # now that we have a schema object we need to check if we need to create or modify and existing index
+      if @db.transaction_active?
+        if exists?(name)
+          load_index(name)
+          do_modify(schema)
+        else
+          do_create(schema)
+        end
+        prepare_statements
+      else
+        @db.transaction(:immediate) do
+          if exists?(name)
+            load_index(name)
+            do_modify(schema)
+          else
+            do_create(schema)
+          end
+          prepare_statements
+        end
+      end
+    elsif exists?(name)
+      load_index(name)
+      prepare_statements
+    # an index already exists, load it from the database and return the index instance to the caller
+    else
+      raise "index does not exist and no schema was supplied"
+    end
+  end
+
+  def load_index(name)
+    # we cannot use get_config_value here since the schema object is not created yet, should we allow something here?
+    @schema = begin
+      Litesearch::Schema.new(Oj.load(@db.get_first_value("SELECT v from #{name}_config where k = ?", :litesearch_schema.to_s)))
+    rescue
+      nil
+    end
+    raise "index configuration not found, either corrupted or not a litesearch index!" if @schema.nil?
+    self
+  end
+
+  def modify
+    schema = Litesearch::Schema.new
+    yield schema
+    schema.schema[:name] = @schema.schema[:name]
+    do_modify(schema)
+  end
+
+  def rebuild!
+    if @db.transaction_active?
+        do_rebuild
+    else
+      @db.transaction(:immediate) { do_rebuild }
+    end
+  end
+
+  def add(document)
+    @stmts[:insert].execute!(document)
+    @db.last_insert_row_id
+  end
+
+  def remove(id)
+    @stmts[:delete].execute!(id)
+  end
+
+  def count(term = nil)
+    if term
+      @stmts[:count].execute!(term)[0][0]
+    else
+      @stmts[:count_all].execute![0][0]
+    end
+  end
+
+  # search options include
+  # limit: how many records to return
+  # offset: start from which record
+  def search(term, options = {})
+    options = DEFAULT_SEARCH_OPTIONS.merge(options)
+    rs = @stmts[:search].execute(term, options[:limit], options[:offset])
+    generate_results(rs)
+  end
+  
+  def similar(id, limit=10)
+    #   pp term = @db.execute(@schema.sql_for(:similarity_query), id)
+    if @schema.schema[:tokenizer] == :trigram
+      # just use the normal similarity approach for now
+      # need to recondisder that for trigram indexes later
+      rs = @stmts[:similar].execute(id, limit)
+    else
+      rs = @stmts[:similar].execute(id, limit)
+    end
+    generate_results(rs)
+  end
+  
+  def clear!
+    @stmts[:delete_all].execute!(id)
+  end
+
+  def drop!
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
+      if @schema.sql_for(:create_secondary_triggers)
+        @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
+      end
+    end
+    @db.execute(@schema.sql_for(:drop))
+  end
+
+  private
+
+  def generate_results(rs)
+    result = []
+    if @db.results_as_hash
+      rs.each_hash do |hash|
+        result << hash
+      end
+    else
+      result = rs.to_a
+    end
+    result    
+  end
+
+  def exists?(name)
+    @db.get_first_value("SELECT count(*) FROM SQLITE_MASTER WHERE name = ? AND type = 'table' AND (sql like '%fts5%' OR sql like '%FTS5%')", name.to_s) == 1
+  end
+
+  def prepare_statements
+    stmt_names = [:insert, :delete, :delete_all, :drop, :count, :count_all, :search, :similar]
+    stmt_names.each do |stmt_name|
+      @stmts[stmt_name] = @db.prepare(@schema.sql_for(stmt_name))
+    end
+  end
+
+  def do_create(schema)
+    @schema = schema
+    @schema.clean
+    # create index
+    @db.execute(schema.sql_for(:create_index, true))
+    @db.execute_batch(schema.sql_for(:create_vocab_tables))
+    # adjust ranking function
+    @db.execute(schema.sql_for(:ranks, true))
+    # create triggers (if any)
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:create_primary_triggers))
+      if (secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers))
+        @db.execute_batch(secondary_triggers_sql)
+      end
+      @db.execute(@schema.sql_for(:rebuild)) if @schema.get(:rebuild_on_create)
+    end
+    set_config_value(:litesearch_schema, @schema.schema)
+  end
+
+  def do_modify(new_schema)
+    changes = @schema.compare(new_schema)
+    # ensure the new schema maintains feild order
+    new_schema.order_fields(@schema)
+    # with the changes object decide what needs to be done to the schema
+    requires_schema_change = false
+    requires_trigger_change = false
+    requires_rebuild = false
+    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:filter_column] || changes[:removed_fields_count] > 0 # any change here will require a schema change
+      requires_schema_change = true
+      # only a change in tokenizer
+      requires_rebuild = changes[:tokenizer] || new_schema.get(:rebuild_on_modify)
+      requires_trigger_change = (changes[:table] || changes[:fields] || changes[:filter_column]) && @schema.get(:type) == :backed
+    end
+    if requires_schema_change
+      # 1. enable schema editing
+      @db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
+      # 2. update the index sql
+      @db.execute(new_schema.sql_for(:update_index), new_schema.sql_for(:create_index))
+      # 3. update the content table sql (if it exists)
+      @db.execute(new_schema.sql_for(:update_content_table), new_schema.sql_for(:create_content_table, new_schema.schema[:fields].count))
+      # adjust shadow tables
+      @db.execute(new_schema.sql_for(:expand_data), changes[:extra_fields_count])
+      @db.execute(new_schema.sql_for(:expand_docsize), changes[:extra_fields_count])
+      @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
+      # need to reprepare statements
+    end
+    if requires_trigger_change
+      @db.execute_batch(new_schema.sql_for(:drop_primary_triggers))
+      @db.execute_batch(new_schema.sql_for(:create_primary_triggers))
+      if (secondary_triggers_sql = new_schema.sql_for(:create_secondary_triggers))
+        @db.execute_batch(new_schema.sql_for(:drop_secondary_triggers))
+        @db.execute_batch(secondary_triggers_sql)
+      end
+    end
+    if changes[:fields] || changes[:table] || changes[:tokenizer] || changes[:weights] || changes[:filter_column]
+      @schema = new_schema
+      set_config_value(:litesearch_schema, @schema.schema)
+      prepare_statements
+      # save_schema
+    end
+    # update the weights if they changed
+    @db.execute(@schema.sql_for(:ranks, true)) if changes[:weights]
+    @db.execute_batch(@schema.sql_for(:create_vocab_tables))
+    do_rebuild if requires_rebuild
+  end
+
+  def do_rebuild
+    # remove any zero weight columns
+    if @schema.get(:type) == :backed
+      @db.execute_batch(@schema.sql_for(:drop_primary_triggers))
+      if (secondary_triggers_sql = @schema.sql_for(:create_secondary_triggers))
+        @db.execute_batch(@schema.sql_for(:drop_secondary_triggers))
+      end
+      @db.execute(@schema.sql_for(:drop))
+      @db.execute(@schema.sql_for(:create_index, true))
+      @db.execute_batch(@schema.sql_for(:create_primary_triggers))
+      @db.execute_batch(secondary_triggers_sql) if secondary_triggers_sql
+      @db.execute(@schema.sql_for(:rebuild))
+    elsif @schema.get(:type) == :standalone
+      removables = []
+      @schema.get(:fields).each_with_index { |f, i| removables << [f[0], i] if f[1][:weight] == 0 }
+      removables.each do |col|
+        @db.execute(@schema.sql_for(:drop_content_col, col[1]))
+        @schema.get(:fields).delete(col[0])
+      end
+      @db.execute("PRAGMA WRITABLE_SCHEMA = TRUE")
+      @db.execute(@schema.sql_for(:update_index), @schema.sql_for(:create_index, true))
+      @db.execute(@schema.sql_for(:update_content_table), @schema.sql_for(:create_content_table, @schema.schema[:fields].count))
+      @db.execute("PRAGMA WRITABLE_SCHEMA = RESET")
+      @db.execute(@schema.sql_for(:rebuild))
+    end
+    @db.execute_batch(@schema.sql_for(:create_vocab_tables))
+    set_config_value(:litesearch_schema, @schema.schema)
+    @db.execute(@schema.sql_for(:ranks, true))
+  end
+
+  def get_config_value(key)
+    Oj.load(@db.get_first_value(@schema.sql_for(:get_config_value), key.to_s)) # rescue nil
+  end
+
+  def set_config_value(key, value)
+    @db.execute(@schema.sql_for(:set_config_value), key.to_s, Oj.dump(value))
+  end
+end