litestack 0.2.6 → 0.4.1

data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb

@@ -0,0 +1,147 @@
+class Litesearch::Schema::BackedAdapter < Litesearch::Schema::ContentlessAdapter
+  
+  private
+
+  def table
+    @schema[:table]
+  end
+
+  def generate_sql
+    super
+    @sql[:rebuild] = :rebuild_sql
+    @sql[:drop_primary_triggers] = :drop_primary_triggers_sql
+    @sql[:drop_secondary_triggers] = :drop_secondary_triggers_sql
+    @sql[:create_primary_triggers] = :create_primary_triggers_sql
+    @sql[:create_secondary_triggers] = :create_secondary_triggers_sql
+  end
+  
+  def drop_primary_triggers_sql
+sql = <<-SQL 
+DROP TRIGGER IF EXISTS #{name}_insert;
+DROP TRIGGER IF EXISTS #{name}_update;
+DROP TRIGGER IF EXISTS #{name}_update_not;
+DROP TRIGGER IF EXISTS #{name}_delete;
+SQL
+  end
+  
+  def create_primary_triggers_sql(active=false)
+    when_stmt = "TRUE"
+    cols = active_cols_names
+    if filter = @schema[:filter_column]
+      when_stmt = "NEW.#{filter} = TRUE"
+      cols << filter
+    end
+
+sql = <<-SQL
+  CREATE TRIGGER #{name}_insert AFTER INSERT ON #{table} WHEN #{when_stmt} BEGIN
+    INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
+  END;
+  CREATE TRIGGER #{name}_update AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN #{when_stmt} BEGIN
+    INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) VALUES (NEW.rowid, #{trigger_cols_sql});
+  END;
+  CREATE TRIGGER #{name}_update_not AFTER UPDATE OF #{cols.join(', ')} ON #{table} WHEN NOT #{when_stmt} BEGIN
+    DELETE FROM #{name} WHERE rowid = NEW.rowid;
+  END;
+  CREATE TRIGGER #{name}_delete AFTER DELETE ON #{table} BEGIN
+    DELETE FROM #{name} WHERE rowid = OLD.id;
+  END;
+SQL
+  end
+  
+  def drop_secondary_trigger_sql(target_table, target_col, col)
+    "DROP TRIGGER IF EXISTS #{target_table}_#{target_col}_#{col}_#{name}_update;"
+  end
+  
+  def create_secondary_trigger_sql(target_table, target_col, col)
+sql = <<-SQL
+CREATE TRIGGER #{target_table}_#{target_col}_#{col}_#{name}_update AFTER UPDATE OF #{target_col} ON #{target_table} BEGIN
+  #{rebuild_sql} AND #{table}.#{col} = NEW.id;
+END;
+SQL
+  end
+
+  def drop_secondary_triggers_sql
+    sql = ""
+    @schema[:fields].each do |name, field|
+      if field[:trigger_sql]
+        sql << drop_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+      end
+    end
+    return sql.empty? ? nil : sql
+  end
+
+  
+  def create_secondary_triggers_sql
+    sql = ""  
+    @schema[:fields].each do |name, field|
+      if field[:trigger_sql]
+        sql << create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+      end
+    end
+    return sql.empty? ? nil : sql
+  end
+  
+  
+  def rebuild_sql
+    conditions = "" 
+    jcs = join_conditions_sql
+    fs = filter_sql
+    conditions = " ON #{jcs} #{fs}" unless jcs.empty? && fs.empty?
+    "INSERT OR REPLACE INTO #{name}(rowid, #{active_field_names.join(', ')}) SELECT #{table}.id, #{select_cols_sql} FROM #{join_tables_sql} #{conditions}" 
+  end
+    
+  def enrich_schema
+    @schema[:fields].each do |name, field|      
+      if field[:target] && ! field[:target].start_with?("#{table}.")  
+        field[:target] = field[:target].downcase
+        target_table, target_col = field[:target].split('.') 
+        field[:col] = "#{name}_id".to_sym unless field[:col]
+        field[:target_table] = target_table.to_sym 
+        field[:target_col] = target_col.to_sym
+        field[:sql] = "(SELECT #{field[:target_col]} FROM #{field[:target_table]} WHERE id = NEW.#{field[:col]})"
+        field[:trigger_sql] = true # create_secondary_trigger_sql(field[:target_table], field[:target_col], field[:col])
+        field[:target_table_alias] = "#{field[:target_table]}_#{name}"
+      else
+        field[:col] = name unless field[:col]
+        field[:sql] = field[:col]
+        field[:target_table] = @schema[:table]
+        field[:target] = "#{@schema[:table]}.#{field[:sql]}"
+      end
+    end
+  end
+  
+  def filter_sql
+    sql = "" 
+    sql << " AND #{@schema[:filter_column]} = TRUE " if @schema[:filter_column]
+    sql
+  end
+
+  def trigger_cols_sql
+    active_fields.collect do |name, field|
+      field[:trigger_sql] ? field[:sql] : "NEW.#{field[:sql]}"
+    end.join(", ")
+  end
+  
+  def select_cols_sql
+    active_fields.collect do |name, field|
+      field[:trigger_sql] != nil ? "#{field[:target_table_alias]}.#{field[:target_col]}" : field[:target]
+    end.join(', ')
+  end
+  
+  def join_tables_sql
+    tables = [@schema[:table]]    
+    active_fields.each do |name, field|
+      tables << "#{field[:target_table]} AS #{field[:target_table_alias]}" if field[:trigger_sql]    
+    end
+    tables.uniq.join(", ")
+  end
+  
+  def join_conditions_sql
+    conditions = []
+    active_fields.each do |name, field|
+      conditions << "#{field[:target_table_alias]}.id = #{@schema[:table]}.#{field[:col]}" if field[:trigger_sql]    
+    end
+    conditions.join(" AND ")    
+  end 
+  
+end

data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb

@@ -0,0 +1,128 @@
+class Litesearch::Schema::BasicAdapter
+  
+  def initialize(schema)
+    @schema = schema
+    @sql = {}
+    enrich_schema
+    generate_sql
+  end
+  
+  def name
+    @schema[:name]
+  end
+  
+  def table
+    @schema[:table]
+  end
+
+  def fields
+    @schema[:fields]
+  end
+
+  def field_names
+    @schema[:fields].keys
+  end
+  
+  def active_fields
+    @schema[:fields].select{|k, v| v[:weight] != 0 }
+  end
+  
+  def active_field_names
+    active_fields.keys
+  end
+  
+  def active_cols_names
+    active_fields.collect{|k, v| v[:col]}
+  end
+  
+  def weights
+    @schema[:fields].values.collect{|v| v[:weight].to_f }
+  end
+
+  def active_weights
+    active_fields.values.collect{|v| v[:weight].to_f }
+  end
+
+  def tokenizer_sql
+    Litesearch::Schema::TOKENIZERS[@schema[:tokenizer]]
+  end
+
+  def order_fields(old_schema)
+    new_fields = {}
+    old_field_names = old_schema.schema[:fields].keys
+    old_field_names.each do |name|
+      new_fields[name] = @schema[:fields].delete(name) 
+    end
+    missing_field_names = field_names - old_field_names
+    missing_field_names.each do |name|
+      new_fields[name] = @schema[:fields].delete(name)
+    end   
+    @schema[:fields] = new_fields # this should be in order now
+    generate_sql
+    enrich_schema
+  end
+
+  def sql_for(method, *args)
+    if sql = @sql[method]
+      if sql.is_a? String
+        return sql
+      elsif sql.is_a? Proc
+        return sql.call(*args)
+      elsif sql.is_a? Symbol
+        return self.send(sql, *args)
+      elsif sql.is_a? Litesearch::SchemaChangeException
+        raise sql
+      end
+    end
+  end
+    
+  def generate_sql
+    @sql[:create_index] = :create_index_sql
+    @sql[:insert] = "INSERT OR REPLACE INTO #{name}(rowid, #{active_col_names_sql}) VALUES (:id, #{active_col_names_var_sql}) RETURNING rowid"
+    @sql[:delete] = "DELETE FROM #{name} WHERE rowid = :id"
+    @sql[:count] = "SELECT count(*) FROM #{name}(:term)"
+    @sql[:count_all] = "SELECT count(*) FROM #{name}"
+    @sql[:delete_all] = "DELETE FROM #{name}"
+    @sql[:drop] = "DROP TABLE #{name}"
+    @sql[:expand_data] = "UPDATE #{name}_data SET block = block || zeroblob(:length) WHERE id = 1"
+    @sql[:expand_docsize] = "UPDATE #{name}_docsize SET sz = sz || zeroblob(:length)"
+    @sql[:ranks] = :ranks_sql 
+    @sql[:set_config_value] = "INSERT OR REPLACE INTO #{name}_config(k, v) VALUES (:key, :value)"
+    @sql[:get_config_value] = "SELECT v FROM #{name}_config WHERE k = :key"
+    @sql[:search] = "SELECT rowid AS id, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
+    @sql[:update_index] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}'"
+    @sql[:update_content_table] = "UPDATE sqlite_schema SET sql = :sql WHERE name = '#{name}_content'"
+  end
+  
+  private
+  
+  def ranks_sql(active=false)
+    if active
+      weights_sql = weights.join(', ')
+    else
+      weights_sql = active_weights.join(', ')      
+    end
+    "INSERT INTO #{name}(#{name}, rank) VALUES ('rank', 'bm25(#{weights_sql})')"
+  end
+  
+  def active_col_names_sql
+    active_field_names.join(', ')
+  end
+  
+  def active_col_names_var_sql
+    ":#{active_field_names.join(', :')}"
+  end
+    
+  def col_names_sql
+    field_names.join(', ')
+  end
+  
+  def col_names_var_sql
+    ":#{field_names.join(', :')}"
+  end
+
+  def enrich_schema
+  end
+    
+end
+

data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb

@@ -0,0 +1,17 @@
+class Litesearch::Schema::ContentlessAdapter < Litesearch::Schema::BasicAdapter
+
+  private
+
+  def generate_sql
+    super
+    #@sql[:rebuild_index] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
+    #@sql[:rebuild] = Litesearch::SchemaChangeException.new("You cannot rebuild a contentless index")
+  end
+  
+  def create_index_sql(active = false)
+    col_names = active ? active_col_names_sql : col_names_sql
+    "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, content='', contentless_delete=1, tokenize='#{tokenizer_sql}')"
+  end
+
+end
+

data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb

@@ -0,0 +1,33 @@
+class Litesearch::Schema::StandaloneAdapter < Litesearch::Schema::BasicAdapter
+  
+  def generate_sql
+    super
+    @sql[:move_content] = "ALTER TABLE #{name}_content RENAME TO #{name}_content_temp"
+    @sql[:adjust_temp_content] = "UPDATE sqlite_schema SET sql (SELECT sql FROM sqlite_schema WHERE name = '#{name}_content') WHERE name = #{name}_content_temp"
+    @sql[:restore_content] = "ALTER TABLE #{name}_content_temp RENAME TO #{name}_content"
+    @sql[:rebuild] = "INSERT INTO #{name}(#{name}) VALUES ('rebuild')"
+    @sql[:drop_content_table] = "DROP TABLE #{name}_content"
+    @sql[:drop_content_col] = :drop_content_col_sql
+    @sql[:create_content_table] = :create_content_table_sql 
+    @sql[:search] = "SELECT rowid AS id, *, -rank AS search_rank FROM #{name}(:term) WHERE rank !=0 ORDER BY rank LIMIT :limit OFFSET :offset"
+  end
+
+  private
+
+  def create_index_sql(active = false)
+    col_names = active ? active_col_names_sql : col_names_sql
+    "CREATE VIRTUAL TABLE #{name} USING FTS5(#{col_names}, tokenize='#{tokenizer_sql}')"
+  end
+    
+  def drop_content_col_sql(col_index)
+    "ALTER TABLE #{name}_content DROP COLUMN c#{col_index}"
+  end
+  
+  def create_content_table_sql(count)
+    cols = [] 
+    count.times{|i| cols << "c#{i}" }
+    "CREATE TABLE #{name}_content(id INTEGER PRIMARY KEY, #{cols.join(', ')})"
+  end
+  
+end
+

data/lib/litestack/litesearch/schema_adapters.rb

@@ -0,0 +1,9 @@
+require_relative './schema_adapters/basic_adapter.rb'
+require_relative './schema_adapters/standalone_adapter.rb'
+require_relative './schema_adapters/contentless_adapter.rb'
+require_relative './schema_adapters/backed_adapter.rb'
+
+
+
+
+

data/lib/litestack/litesearch.rb

@@ -0,0 +1,37 @@
+module Litesearch
+  class Index; end
+  class Schema; end
+end
+
+require_relative './litesearch/index'
+require_relative './litesearch/model'
+
+module Litesearch
+  
+  def litesearch_index_cache
+    @litesearch_index_cache ||= {}
+  end
+
+  def search_index(name)
+    # normalize the index name
+    # find the index in the db cache
+    name = name.to_s.downcase.to_sym
+    index = litesearch_index_cache[name]
+    # if the index is in the cache and no block is given then return it
+    return index if index && !block_given?
+    # if either there is no index in the cache or a block is given
+    # create a new index instance and then place it in the cache and return
+    if block_given?
+      index = Index.new(self, name) do |schema|
+        yield schema
+        schema.name(name)
+      end
+    else
+      index = Index.new(self, name)
+    end
+    litesearch_index_cache[name] = index
+  end
+
+end
+
+

data/lib/litestack/litesupport.rb

@@ -7,14 +7,11 @@ require 'yaml'
 require 'pathname'
 require 'fileutils'
 
+require_relative "./litescheduler"
+
 module Litesupport
 
   class Error < StandardError; end
-  
-  def self.max_contexts
-    return 50 if scheduler == :fiber || scheduler == :polyphony
-    5    
-  end
 
   # Detect the Rack or Rails environment.
   def self.detect_environment
@@ -23,7 +20,7 @@ module Litesupport
     elsif ENV["RACK_ENV"]
       ENV["RACK_ENV"]
     elsif ENV["APP_ENV"]
-      ENV["RACK_ENV"]
+      ENV["APP_ENV"]
     else
       "development"
     end
@@ -32,89 +29,11 @@ module Litesupport
   def self.environment
     @environment ||= detect_environment
   end
-
-  # cache the scheduler we are running in
-  # it is an error to change the scheduler for a process
-  # or for a child forked from that process
-  def self.scheduler
-    @scehduler ||= detect_scheduler
-  end
-
-  # identify which scheduler we are running in
-  # we currently support :fiber, :polyphony, :iodine & :threaded
-  # in the future we might want to expand to other schedulers
-  def self.detect_scheduler
-    return :fiber if Fiber.scheduler 
-    return :polyphony if defined? Polyphony
-    return :iodine if defined? Iodine
-    return :threaded # fall back for all other schedulers
-  end
-  
-  # spawn a new execution context
-  def self.spawn(&block)
-    if self.scheduler == :fiber
-      Fiber.schedule(&block)
-    elsif self.scheduler == :polyphony
-      spin(&block)
-    elsif self.scheduler == :threaded or self.scheduler == :iodine
-      Thread.new(&block)
-    end
-    # we should never reach here
-  end
-
-  def self.context
-    if scheduler == :fiber || scheduler == :poylphony
-      Fiber.current.storage
-    else
-      Thread.current
-    end
-  end
-  
-  def self.current_context
-    if scheduler == :fiber || scheduler == :poylphony
-      Fiber.current
-    else
-      Thread.current
-    end
-  end
-  
-  # switch the execution context to allow others to run
-  def self.switch
-    if self.scheduler == :fiber
-      Fiber.scheduler.yield
-      true
-    elsif self.scheduler == :polyphony
-      Fiber.current.schedule
-      Thread.current.switch_fiber
-      true
-    else
-      #Thread.pass
-      false
-    end   
-  end
-  
-  # mutex initialization
-  def self.mutex
-    # a single mutex per process (is that ok?)
-    @@mutex ||= Mutex.new
-  end
-  
-  # bold assumption, we will only synchronize threaded code
-  # if some code explicitly wants to synchronize a fiber
-  # they must send (true) as a parameter to this method
-  # else it is a no-op for fibers
-  def self.synchronize(fiber_sync = false, &block)
-    if self.scheduler == :fiber or self.scheduler == :polyphony
-      yield # do nothing, just run the block as is
-    else
-      self.mutex.synchronize(&block)
-    end
-  end
   
   # common db object options
   def self.create_db(path)
     db = SQLite3::Database.new(path)
-    db.busy_handler{ switch || sleep(0.0001) }
+    db.busy_handler{ Litescheduler.switch || sleep(0.0001) }
     db.journal_mode = "WAL"
     db.instance_variable_set(:@stmts, {})
     class << db
@@ -153,7 +72,7 @@ module Litesupport
     end
     
     def synchronize(&block)
-      if Litesupport.scheduler == :threaded || Litesupport.scheduler == :iodine
+      if Litescheduler.backend == :threaded || Litescheduler.backend == :iodine
         @mutex.synchronize{ block.call }
       else
         block.call
@@ -162,57 +81,32 @@ module Litesupport
   
   end
    
-  module Forkable
-    
-    def _fork(*args)
-      ppid = Process.pid
-      result = super      
-      if Process.pid != ppid
-        # trigger a restart of all connections owned by Litesupport::Pool
-      end
-      result
-    end
-    
-  end
-
-  #::Process.singleton_class.prepend(::Litesupport::Forkable)
-  
   class Pool
   
     def initialize(count, &block)
       @count = count
       @block = block
-      @resources = []
+      @resources = Thread::Queue.new
       @mutex = Litesupport::Mutex.new
       @count.times do
         resource = @mutex.synchronize{ block.call }
-        @resources << [resource, :free]
+        @resources << resource 
       end
     end
     
     def acquire
-      # check for pid changes
-      acquired = false
       result = nil
-      while !acquired do
-        @mutex.synchronize do
-          if resource = @resources.find{|r| r[1] == :free }
-            resource[1] = :busy
-            begin
-              result = yield resource[0]
-            rescue Exception => e
-              raise e
-            ensure
-              resource[1] = :free
-              acquired = true
-            end
-          end
-        end
-        sleep 0.001 unless acquired
+      resource = @resources.pop
+      begin
+        result = yield resource
+      rescue Exception => e
+        raise e
+      ensure
+        @resources << resource
       end
       result
     end
-    
+        
   end
      
   module ForkListener
@@ -230,7 +124,7 @@ module Litesupport
     def _fork(*args)
       ppid = Process.pid
       result = super
-      if Process.pid != ppid && [:threaded, :iodine].include?(Litesupport.scheduler)
+      if Process.pid != ppid && [:threaded, :iodine].include?(Litescheduler.backend)
         ForkListener.listeners.each{|l| l.call }
       end
       result
@@ -333,15 +227,20 @@ module Litesupport
     def run_method(method, *args)
       @conn.acquire{|q| q.send(method, *args)}
     end
+
+    def run_stmt_method(stmt, method, *args)
+      @conn.acquire{|q| q.stmts[stmt].send(method, *args)}
+    end
+
     
     def create_pooled_connection(count = 1)
       Litesupport::Pool.new(1){create_connection}  
     end
 
     # common db object options
-    def create_connection
+    def create_connection(path_to_sql_file = nil)
       conn = SQLite3::Database.new(@options[:path])
-      conn.busy_handler{ Litesupport.switch || sleep(rand * 0.002) }
+      conn.busy_handler{ Litescheduler.switch || sleep(rand * 0.002) }
       conn.journal_mode = "WAL"
       conn.synchronous = @options[:sync] || 1
       conn.mmap_size = @options[:mmap_size] || 0
@@ -349,9 +248,40 @@ module Litesupport
       class << conn
         attr_reader :stmts
       end
+      yield conn if block_given?
+      # use the <client>.sql.yml file to define the schema and compile prepared statements
+      unless path_to_sql_file.nil?
+        sql = YAML.load_file(path_to_sql_file)
+        version = conn.get_first_value("PRAGMA user_version")
+        sql["schema"].each_pair do |v, obj| 
+          if v > version
+            conn.transaction do 
+              obj.each do |k, s| 
+                begin
+                  conn.execute(s)
+                rescue Exception => e
+                  STDERR.puts "Error parsing #{k}"
+                  STDERR.puts s
+                  raise e               
+                end
+              end
+              conn.user_version = v
+            end
+          end
+        end 
+        sql["stmts"].each do |k, v| 
+          begin
+            conn.stmts[k.to_sym] = conn.prepare(v)
+          rescue Exception => e
+            STDERR.puts "Error parsing #{k}"
+            STDERR.puts v
+            raise e               
+          end
+        end
+      end
       conn
     end
-    
+        
   end
       
 end  

data/lib/litestack/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litestack
-  VERSION = "0.2.6"
+  VERSION = "0.4.1"
 end       

data/lib/litestack.rb

@@ -2,6 +2,7 @@
 
 # load core classes
 require_relative "./litestack/version"
+require_relative "./litestack/litescheduler"
 require_relative "./litestack/litesupport"
 require_relative "./litestack/litemetric"
 require_relative "./litestack/litedb"
@@ -16,7 +17,7 @@ require_relative "./railties/rails/commands/dbconsole" if defined? Rails && defi
 require_relative "./active_support/cache/litecache" if defined? ActiveSupport
 require_relative "./active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
 require_relative "./action_cable/subscription_adapter/litecable" if defined? ActionCable
-require_relative "./litestack/railtie" if defined? Rails
+require_relative "./litestack/railtie" if defined? Rails::Railtie
 
 module Litestack
   class NotImplementedError < Exception; end

data/lib/sequel/adapters/litedb.rb

@@ -1,10 +1,10 @@
 require_relative '../../litestack/litedb'
 require 'sequel'
 require 'sequel/adapters/sqlite'
-#require 'shared/litedb'
 
 module Sequel
   module Litedb
+  
   	include SQLite
 
     LITEDB_TYPES = SQLITE_TYPES
@@ -15,7 +15,7 @@ module Sequel
       
       def connect(server)
       
-        Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litesupport.scheduler
+        Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litescheduler.backend
       
         opts = server_opts(server)
         opts[:database] = ':memory:' if blank_object?(opts[:database])
@@ -36,6 +36,7 @@ module Sequel
         end
         
         db.instance_variable_set(:@prepared_statements, {})
+        @raw_db = db
         db
       end