litestack 0.4.2 → 0.4.4

data/lib/litestack/litecache.sql.yml

@@ -3,10 +3,10 @@ schema:
     create_table_data: >
       CREATE TABLE IF NOT EXISTS data(
         id TEXT PRIMARY KEY,
-        value TEXT,
+        value ANY,
         expires_in INTEGER,
         last_used INTEGER
-      );
+      ) STRICT;
     create_expiry_index: >
       CREATE INDEX IF NOT EXISTS expiry_index ON data (expires_in);
     create_last_used_index: >
@@ -14,7 +14,7 @@ schema:
 
 stmts:
   pruner: >
-    DELETE FROM data WHERE expires_in <= $1;
+    DELETE FROM data WHERE expires_in <= unixepoch('now');
 
   extra_pruner: >
     DELETE FROM data WHERE id IN (
@@ -49,7 +49,7 @@ stmts:
       value = EXCLUDED.value,
       last_used = EXCLUDED.last_used,
       expires_in = EXCLUDED.expires_in;
-
+  
   inserter: >
     INSERT INTO data (id, value, expires_in, last_used)
     VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now'))
@@ -65,7 +65,7 @@ stmts:
     SELECT id FROM data WHERE id = $1;
   
   getter: >
-    SELECT id, value, expires_in FROM data WHERE id = $1;
+    SELECT id, value, expires_in FROM data WHERE id = $1 AND expires_in >= unixepoch('now');
   
   deleter: >
     delete FROM data WHERE id = $1 RETURNING value;
@@ -79,6 +79,8 @@ stmts:
       last_used = EXCLUDED.last_used,
       expires_in = EXCLUDED.expires_in;
   
+      
+  
   counter: >
     SELECT count(*) FROM data;
   

data/lib/litestack/litedb.rb

@@ -15,7 +15,7 @@ class Litedb < ::SQLite3::Database
   # add litesearch support
   include Litesearch
 
-  # overrride the original initilaizer to allow for connection configuration
+  # override the original initilaizer to allow for connection configuration
   def initialize(file, options = {}, zfs = nil)
     if block_given?
       super(file, options, zfs) do |db|
@@ -31,6 +31,10 @@ class Litedb < ::SQLite3::Database
     collect_metrics if @collecting_metrics
   end
 
+  def sqlite_version
+    SQLite3::SQLITE_VERSION_NUMBER
+  end
+
   def collecting_metrics?
     @collecting_metrics
   end

data/lib/litestack/litejob.rb

@@ -1,6 +1,6 @@
 # frozen_stringe_literal: true
 
-require_relative "./litejobqueue"
+require_relative "litejobqueue"
 
 ##
 # Litejob is a Ruby module that enables seamless integration of the Litejobqueue job queueing system into Ruby applications. By including the Litejob module in a class and implementing the #perform method, developers can easily enqueue and process jobs asynchronously.

data/lib/litestack/litejobqueue.rb

@@ -1,7 +1,7 @@
 # frozen_stringe_literal: true
 
-require_relative "./litequeue"
-require_relative "./litemetric"
+require_relative "litequeue"
+require_relative "litemetric"
 
 ##
 # Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
@@ -15,7 +15,7 @@ class Litejobqueue < Litequeue
   include Litemetric::Measurable
 
   # the default options for the job queue
-  # can be overriden by passing new options in a hash
+  # can be overridden by passing new options in a hash
   # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
   #   config_path: "./litejob.yml" -> were to find the configuration file (if any)
   #   path: "./db/queue.db"
@@ -136,20 +136,28 @@ class Litejobqueue < Litequeue
     # @@queue = nil
     close
   end
-
+  
   private
+  
+  def prepare_search_options(opts)
+    sql_opts = super(opts)
+    sql_opts[:klass] = opts[:klass]
+    sql_opts[:params] = opts[:params]
+    sql_opts  
+  end
 
   def exit_callback
     @running = false # stop all workers
-    return unless @jobs_in_flight > 0
-    puts "--- Litejob detected an exit, cleaning up"
-    index = 0
-    while @jobs_in_flight > 0 && index < 30 # 3 seconds grace period for jobs to finish
-      puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
-      sleep 0.1
-      index += 1
+    if @jobs_in_flight > 0
+      puts "--- Litejob detected an exit, cleaning up"
+      index = 0
+      while @jobs_in_flight > 0 && index < 30 # 3 seconds grace period for jobs to finish
+        puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
+        sleep 0.1
+        index += 1
+      end
+      puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
     end
-    puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
   end
 
   def setup
@@ -179,6 +187,8 @@ class Litejobqueue < Litequeue
 
   # create a worker according to environment
   def create_worker
+    # temporarily stop this feature until a better solution is implemented
+    #return if defined?(Rails) && !defined?(Rails::Server)
     Litescheduler.spawn do
       worker_sleep_index = 0
       while @running
@@ -186,7 +196,7 @@ class Litejobqueue < Litequeue
         @queues.each do |priority, queues| # iterate through the levels
           queues.each do |queue, spawns| # iterate through the queues in the level
             batched = 0
-
+            
             while (batched < priority) && (payload = pop(queue, 1)) # fearlessly use the same queue object
               capture(:dequeue, queue)
               processed += 1
@@ -201,7 +211,7 @@ class Litejobqueue < Litequeue
         end
         if processed == 0
           sleep @options[:sleep_intervals][worker_sleep_index]
-          worker_sleep_index += 1 if worker_sleep_index < @options[:sleep_intervals].length - 1
+          worker_sleep_index += 1 if worker_sleep_index < (@options[:sleep_intervals].length - 1)
         else
           worker_sleep_index = 0 # reset the index
         end

data/lib/litestack/litemetric.rb

@@ -2,7 +2,7 @@
 
 require "singleton"
 
-require_relative "./litesupport"
+require_relative "litesupport"
 
 # this class is a singleton
 # and should remain so
@@ -148,10 +148,11 @@ class Litemetric
   end
 
   def exit_callback
-    return unless @collector.count > 0
-    warn "--- Litemetric detected an exit, flushing metrics"
     @running = false
-    @collector.flush
+    if @collector.count > 0
+      warn "--- Litemetric detected an exit, flushing metrics"
+      @collector.flush
+    end
   end
 
   def setup
@@ -203,8 +204,8 @@ class Litemetric
     def create_snapshotter
       Litescheduler.spawn do
         while @running
-          sleep @litemetric.options[:snapshot_interval]
           capture_snapshot
+          sleep @litemetric.options[:snapshot_interval]
         end
       end
     end
@@ -275,7 +276,7 @@ class Litemetric
     def capture_single_key(topic, event, key, value, time = nil)
       run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, time, 1, value)
     end
-    
+
     def count
       run_stmt(:event_count)[0][0]
     end

data/lib/litestack/litemetric.sql.yml

@@ -19,7 +19,7 @@ schema:
         created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL,
         resolution TEXT DEFAULT('minute') NOT NULL,
         PRIMARY KEY(resolution, created_at, topic, name, key)
-      ) STRICT;
+      );
 
     create_topic_index_on_events: >
       CREATE INDEX events_topic_index ON events (resolution, created_at, topic) WHERE name = '___';

data/lib/litestack/litemetric_collector.sql.yml

@@ -12,7 +12,7 @@ schema:
         created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE,
         resolution TEXT DEFAULT('minute') NOT NULL,
         PRIMARY KEY(resolution, created_at, topic, name, key)
-      ) STRICT;
+      );
 
 stmts:
   capture_event: >

data/lib/litestack/litequeue.rb

@@ -13,7 +13,7 @@ require_relative "litesupport"
 
 class Litequeue
   # the default options for the queue
-  # can be overriden by passing new options in a hash
+  # can be overridden by passing new options in a hash
   # to Litequeue.new
   #   path: "./queue.db"
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
@@ -80,7 +80,7 @@ class Litequeue
     run_sql("DELETE FROM queue WHERE iif(?1 IS NOT NULL, name = ?1,  TRUE)", queue)
   end
 
-  # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
+  # returns a count of entries in all queues, or if a queue name is given, returns the count of entries in that queue
   def count(queue = nil)
     run_sql("SELECT count(*) FROM queue WHERE iif(?1 IS NOT NULL, name = ?1, TRUE)", queue)[0][0]
   end
@@ -111,9 +111,24 @@ class Litequeue
       queues: queues
     }
   end
+  
+  def find(opts = {})
+    run_stmt(:search, prepare_search_options(opts))
+  end
 
   private
 
+  def prepare_search_options(opts)
+    sql_opts = {}
+    sql_opts[:fire_at_from] = opts[:fire_at][0] rescue nil
+    sql_opts[:fire_at_to] = opts[:fire_at][1] rescue nil
+    sql_opts[:created_at_from] = opts[:created_at][0] rescue nil
+    sql_opts[:created_at_to] = opts[:created_at][1] rescue nil
+    sql_opts[:name] = opts[:queue]
+    sql_opts[:dir] = opts[:dir] == :desc ? -1 : 1
+    sql_opts
+  end
+
   def create_connection
     super("#{__dir__}/litequeue.sql.yml") do |conn|
       conn.wal_autocheckpoint = 10000

data/lib/litestack/litequeue.sql.yml

@@ -4,7 +4,7 @@ schema:
       CREATE TABLE IF NOT EXISTS queue(
         id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE,
         name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE,
-        fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE,
+        fire_at REAL DEFAULT(unixepoch('subsec')) NOT NULL ON CONFLICT REPLACE,
         value TEXT,
         created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE
       ) WITHOUT ROWID;
@@ -16,21 +16,36 @@ stmts:
 
   push: >
     INSERT INTO queue(id, name, fire_at, value)
-    VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3)
+    VALUES (hex(randomblob(32)), $1, (unixepoch('subsec') + $2), $3)
     RETURNING id, name;
 
   repush: >
     INSERT INTO queue(id, name, fire_at, value)
-    VALUES (?, ?, (unixepoch() + ?), ?)
+    VALUES (?, ?, (unixepoch('subsec') + ?), ?)
     RETURNING name;
 
+  pop_in_place: >
+    UPDATE queue
+    SET 
+      name = '->:' || name,
+      fire_at = unixepoch('subsec')
+    WHERE (name, fire_at, id)
+    IN (
+      SELECT name, fire_at, id FROM queue
+      WHERE name = ifnull($1, 'default')
+        AND fire_at <= (unixepoch('subsec'))
+      ORDER BY fire_at ASC
+      LIMIT ifnull($2, 1)
+    )
+    RETURNING id, value;      
+
   pop: >
     DELETE FROM queue
     WHERE (name, fire_at, id)
     IN (
       SELECT name, fire_at, id FROM queue
       WHERE name = ifnull($1, 'default')
-        AND fire_at <= (unixepoch())
+        AND fire_at <= (unixepoch('subsec'))
       ORDER BY fire_at ASC
       LIMIT ifnull($2, 1)
     )
@@ -40,7 +55,25 @@ stmts:
     DELETE FROM queue
     WHERE id = $1
     RETURNING value;
-
+  
+  search: >
+    SELECT *,  :params, value ->> '$.params', value ->> '$.params' LIKE '%'||'three_second'||'%' FROM queue 
+    WHERE
+      name = ifnull(:name, name)
+    AND
+      iif(:fire_at_from, fire_at >= :fire_at_from, true)
+    AND
+      iif(:fire_at_to, fire_at <= :fire_at_to, true)
+    AND
+      iif(:created_at_from, created_at >= :created_at_from, true)
+    AND
+      iif(:created_at_to, created_at <= :created_at_to, true)
+    AND
+      iif(:klass IS NOT NULL, value ->> '$.klass' LIKE '%'||:klass||'%', true)  
+    AND
+      iif(:params IS NOT NULL, value ->> '$.params' LIKE '%'||:params||'%', true)  
+    ORDER BY created_at * :dir;
+    
   info: >
     SELECT
       name,

data/lib/litestack/litescheduler.rb

@@ -29,7 +29,7 @@ module Litescheduler
   end
 
   def self.storage
-    if backend == :fiber || backend == :poylphony
+    if fiber_backed?
       Fiber.current.storage
     else
       Thread.current
@@ -37,7 +37,7 @@ module Litescheduler
   end
 
   def self.current
-    if backend == :fiber || backend == :poylphony
+    if fiber_backed?
       Fiber.current
     else
       Thread.current
@@ -64,7 +64,7 @@ module Litescheduler
   # they must send (true) as a parameter to this method
   # else it is a no-op for fibers
   def self.synchronize(fiber_sync = false, &block)
-    if (backend == :fiber) || (backend == :polyphony)
+    if fiber_backed?
       yield # do nothing, just run the block as is
     else
       mutex.synchronize(&block)
@@ -72,7 +72,7 @@ module Litescheduler
   end
 
   def self.max_contexts
-    return 50 if backend == :fiber || backend == :polyphony
+    return 50 if fiber_backed?
     5
   end
 
@@ -81,4 +81,9 @@ module Litescheduler
     # a single mutex per process (is that ok?)
     @@mutex ||= Mutex.new
   end
+
+  def self.fiber_backed?
+    backend == :fiber || backend == :polyphony
+  end
+  private_class_method :fiber_backed?
 end

data/lib/litestack/litesearch/index.rb

@@ -1,5 +1,5 @@
 require "oj"
-require_relative "./schema"
+require_relative "schema"
 
 class Litesearch::Index
   DEFAULT_SEARCH_OPTIONS = {limit: 25, offset: 0}
@@ -71,7 +71,7 @@ class Litesearch::Index
 
   def rebuild!
     if @db.transaction_active?
-        do_rebuild
+      do_rebuild
     else
       @db.transaction(:immediate) { do_rebuild }
     end
@@ -102,19 +102,20 @@ class Litesearch::Index
     rs = @stmts[:search].execute(term, options[:limit], options[:offset])
     generate_results(rs)
   end
-  
-  def similar(id, limit=10)
+
+  def similar(id, limit = 10)
     #   pp term = @db.execute(@schema.sql_for(:similarity_query), id)
-    if @schema.schema[:tokenizer] == :trigram
+    rs = if @schema.schema[:tokenizer] == :trigram
       # just use the normal similarity approach for now
       # need to recondisder that for trigram indexes later
-      rs = @stmts[:similar].execute(id, limit)
+      @stmts[:similar].execute(id, limit) # standard:disable Style/IdenticalConditionalBranches
     else
-      rs = @stmts[:similar].execute(id, limit)
+      @stmts[:similar].execute(id, limit) # standard:disable Style/IdenticalConditionalBranches
     end
+
     generate_results(rs)
   end
-  
+
   def clear!
     @stmts[:delete_all].execute!(id)
   end
@@ -140,7 +141,7 @@ class Litesearch::Index
     else
       result = rs.to_a
     end
-    result    
+    result
   end
 
   def exists?(name)
@@ -175,7 +176,7 @@ class Litesearch::Index
 
   def do_modify(new_schema)
     changes = @schema.compare(new_schema)
-    # ensure the new schema maintains feild order
+    # ensure the new schema maintains field order
     new_schema.order_fields(@schema)
     # with the changes object decide what needs to be done to the schema
     requires_schema_change = false

data/lib/litestack/litesearch/model.rb

@@ -11,6 +11,7 @@ module Litesearch::Model
       klass.include Litesearch::Model::ActiveRecordInstanceMethods
       klass.extend Litesearch::Model::ActiveRecordClassMethods
       ActiveRecord::Base.extend Litesearch::Model::BaseClassMethods
+      Litesearch::Schema.prepend Litesearch::Model::ActiveRecordSchemaMethods
     end
   end
 
@@ -21,7 +22,7 @@ module Litesearch::Model
   end
 
   module InstanceMethods
-    def similar(limit=10)
+    def similar(limit = 10)
       conn = self.class.get_connection
       idx = conn.search_index(self.class.send(:index_name))
       r_a_h = conn.results_as_hash
@@ -36,14 +37,38 @@ module Litesearch::Model
       end
       result
     end
-        
   end
 
   module ClassMethods
+  
     def litesearch
+      # it is possible that this code is running when there is no table created yet
+      if !defined?(ActiveRecord::Base).nil? && ancestors.include?(ActiveRecord::Base)
+        unless table_exists?
+          # capture the schema block
+          @schema = ::Litesearch::Schema.new
+          @schema.model_class = self if @schema.respond_to? :model_class
+          @schema.type :backed 
+          @schema.table table_name.to_sym
+          yield @schema
+          @schema.post_init
+          @schema_not_created = true
+          after_initialize do
+            if self.class.instance_variable_get(:@schema_not_created)
+              self.class.get_connection.search_index(self.class.index_name) do |schema|
+                @schema.model_class = self.class if @schema.respond_to? :model_class
+                schema.merge(self.class.instance_variable_get(:@schema))
+              end
+              self.class.instance_variable_set(:@schema_not_created, false)
+            end
+          end
+          return nil
+        end
+      end
       idx = get_connection.search_index(index_name) do |schema|
         schema.type :backed
         schema.table table_name.to_sym
+        schema.model_class = self if schema.respond_to? :model_class
         yield schema
         schema.post_init
         @schema = schema # save the schema
@@ -77,7 +102,7 @@ module Litesearch::Model
       else
         models_hash = search_models
       end
-      # remove the models from the options hash before passing it ot the query
+      # remove the models from the options hash before passing it to the query
       options.delete(:models)
       models_hash.each do |name, klass|
         selects << "SELECT '#{name}' AS model, rowid, -rank AS search_rank FROM #{index_name_for_table(klass.table_name)}(:term)"
@@ -109,6 +134,32 @@ module Litesearch::Model
     end
   end
 
+  module ActiveRecordSchemaMethods
+        
+    attr_accessor :model_class
+    
+    def field(name, attributes = {})
+      keys = attributes.keys
+      if keys.include?(:action_text) || keys.include?(:rich_text)
+        attributes[:source] = "#{ActionText::RichText.table_name}.body" rescue "action_text_rich_texts.body"
+        attributes[:reference] =  :record_id
+        attributes[:conditions] = { record_type:  model_class.name }
+        attributes[:target] = nil
+      elsif keys.include? :as
+        attributes[:source] = attributes[:target] unless attributes[:source] 
+        attributes[:reference] = "#{attributes[:as]}_id"
+        attributes[:conditions] = {"#{attributes[:as]}_type".to_sym => model_class.name }
+        attributes[:target] = nil
+      end
+      super(name, attributes)
+    end    
+
+  def allowed_attributes
+    super + [:polymorphic, :as, :action_text]
+  end
+ 
+  end
+
   module ActiveRecordInstanceMethods; end
 
   module ActiveRecordClassMethods
@@ -121,6 +172,13 @@ module Litesearch::Model
     end
 
     def search(term)
+      if @schema_not_created
+        get_connection.search_index(index_name) do |schema|
+          schema.merge(@schema)
+          schema.model_class = self if schema.respond_to? :model_class
+        end
+        @schema_not_created = false
+      end
       self.select(
         "#{table_name}.*"
       ).joins(

data/lib/litestack/litesearch/schema.rb

@@ -1,6 +1,7 @@
-require_relative "./schema_adapters"
+require_relative "schema_adapters"
 
 class Litesearch::Schema
+
   TOKENIZERS = {
     porter: "porter unicode61 remove_diacritics 2",
     unicode: "unicode61 remove_diacritics 2",
@@ -34,6 +35,10 @@ class Litesearch::Schema
     @schema[:fields] = {} unless @schema[:fields]
   end
 
+  def merge(other_schema)
+    @schema.merge!(other_schema.schema)
+  end
+
   # schema definition API
   def name(new_name)
     @schema[:name] = new_name
@@ -181,7 +186,7 @@ class Litesearch::Schema
   end
 
   def allowed_attributes
-    [:weight, :col, :target]
+    [:weight, :col, :target, :source, :conditions, :reference]
   end
 end