litestack 0.1.8 → 0.2.1

data/lib/litestack/litequeue.rb

@@ -3,6 +3,8 @@
 # all components should require the support module
 require_relative 'litesupport'
 
+#require 'securerandom'
+
 ##
 #Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
 #
@@ -18,10 +20,12 @@ class Litequeue
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
   #   sync: 1 -> sync only when checkpointing
   
+  include Litesupport::Liteconnection
+  
   DEFAULT_OPTIONS = {
     path: "./queue.db",
     mmap_size: 32 * 1024,
-    sync: 1
+    sync: 0
   }
 
   # create a new instance of the litequeue object
@@ -33,8 +37,7 @@ class Litequeue
   #   queue.pop # => "somevalue"
   
   def initialize(options = {})
-    @options = DEFAULT_OPTIONS.merge(options)
-    @queue = Litesupport::Pool.new(1){create_db} # delegate the db creation to the litepool
+    init(options)
   end
    
   # push an item to the queue, optionally specifying the queue name (defaults to default) and after how many seconds it should be ready to pop (defaults to zero)
@@ -45,15 +48,20 @@ class Litequeue
     # also bring back the synchronize block, to prevent
     # a race condition if a thread hits the busy handler
     # before the current thread proceeds after a backoff
-    result = @queue.acquire { |q| q.stmts[:push].execute!(queue, delay, value)[0] }
-    return result[0] if result
+    #id = SecureRandom.uuid # this is somehow expensive, can we improve?
+    run_stmt(:push, queue, delay, value)[0]
+  end
+  
+  def repush(id, value, delay=0, queue='default')
+    run_stmt(:repush, id, queue, delay, value)[0]
   end
   
   alias_method :"<<", :push
+  alias_method :"<<<", :repush
   
   # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
   def pop(queue='default', limit = 1)
-   res = @queue.acquire {|q| res = q.stmts[:pop].execute!(queue, limit) }
+   res = run_stmt(:pop, queue, limit)
    return res[0] if res.length == 1
    return nil if res.empty?
    res
@@ -64,49 +72,61 @@ class Litequeue
   #   id = queue.push("somevalue")
   #   queue.delete(id) # => "somevalue"
   #   queue.pop # => nil
-  def delete(id, queue='default')
-    fire_at, id = id.split("-")
-    result = @queue.acquire{|q| q.stmts[:delete].execute!(queue, fire_at.to_i, id)[0] } 
+  def delete(id)
+    result = run_stmt(:delete, id)[0]  
   end
   
   # deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
   def clear(queue=nil)
-    @queue.acquire{|q| q.execute("DELETE FROM _ul_queue_ WHERE iif(?, queue = ?,  1)", queue) }
+    run_sql("DELETE FROM queue WHERE iif(?, name = ?,  1)", queue)
   end
 
   # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
   def count(queue=nil)
-    @queue.acquire{|q| q.get_first_value("SELECT count(*) FROM _ul_queue_ WHERE iif(?, queue = ?, 1)", queue) }
+    run_sql("SELECT count(*) FROM queue WHERE iif(?, name = ?, 1)", queue)[0][0]
   end
   
   # return the size of the queue file on disk
   def size
-    @queue.acquire{|q| q.get_first_value("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count") }
+    run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0] 
+  end
+  
+  def queues_info
+    run_sql("SELECT name, count(*) AS count, avg(unixepoch() - created_at), min(unixepoch() - created_at), max(unixepoch() - created_at) FROM queue GROUP BY name ORDER BY count DESC ")
   end
   
-  def close
-    @queue.acquire do |q| 
-      q.stmts.each_pair {|k, v| q.stmts[k].close }
-      q.close
+  def info
+    counts = {}
+    queues_info.each do |qc|
+      counts[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
     end
+    {size: size, count: count, info: counts}
   end
 
   private  
-  
-  def create_db
-    db = Litesupport.create_db(@options[:path])
-    db.synchronous = @options[:sync]
-    db.wal_autocheckpoint = 10000 
-    db.mmap_size = @options[:mmap_size]
-    db.execute("CREATE TABLE IF NOT EXISTS _ul_queue_(queue TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE, fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, id TEXT DEFAULT(hex(randomblob(8)) || (strftime('%f') * 100)) NOT NULL ON CONFLICT REPLACE, value TEXT, created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, PRIMARY KEY(queue, fire_at ASC, id) ) WITHOUT ROWID")
-    db.stmts[:push] = db.prepare("INSERT INTO _ul_queue_(queue, fire_at, value) VALUES ($1, (strftime('%s') + $2), $3) RETURNING fire_at || '-' || id")
-    db.stmts[:pop] = db.prepare("DELETE FROM _ul_queue_ WHERE (queue, fire_at, id) IN (SELECT queue, fire_at, id FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at <= (unixepoch()) ORDER BY fire_at ASC LIMIT ifnull($2, 1)) RETURNING fire_at || '-' || id, value")
-    db.stmts[:delete] = db.prepare("DELETE FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at = $2 AND id = $3 RETURNING value")
-    db
+      
+  def create_connection
+    conn = super
+    conn.wal_autocheckpoint = 10000 
+    sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
+    version = conn.get_first_value("PRAGMA user_version")
+    sql["schema"].each_pair do |v, obj| 
+      if v > version
+        conn.transaction(:immediate) do 
+          obj.each{|k, s| conn.execute(s)}
+          conn.user_version = v
+        end
+      end
+    end 
+    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
+    # check if there is an old database and convert entries to the new format
+    if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
+      conn.transaction(:immediate) do
+        conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
+        conn.execute("DROP TABLE _ul_queue_")
+      end
+    end
+    conn
   end
 
-
 end  
-
-
-

data/lib/litestack/litequeue.sql.yml

@@ -0,0 +1,34 @@
+schema:
+  1:    
+    create_table_queue: >  
+      CREATE TABLE IF NOT EXISTS queue(
+        id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE, 
+        name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE, 
+        fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE,
+        value TEXT,
+        created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE
+      ) WITHOUT ROWID 
+      
+    create_index_queue_by_name: >
+      CREATE INDEX IF NOT EXISTS idx_queue_by_name ON queue(name, fire_at ASC)
+
+stmts:
+
+  push: INSERT INTO queue(id, name, fire_at, value) VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3) RETURNING id, name
+  
+  repush: INSERT INTO queue(id, name, fire_at, value) VALUES (?, ?, (unixepoch() + ?), ?) RETURNING name 
+
+  pop: >
+    DELETE FROM queue 
+    WHERE (name, fire_at, id) 
+    IN (
+        SELECT name, fire_at, id FROM queue 
+        WHERE name = ifnull($1, 'default') 
+        AND fire_at <= (unixepoch()) 
+        ORDER BY fire_at ASC 
+        LIMIT ifnull($2, 1)
+    ) 
+    RETURNING id, value
+  
+  delete: DELETE FROM queue WHERE id = $1 RETURNING value
+

data/lib/litestack/litesupport.rb

@@ -1,4 +1,9 @@
+# frozen_stringe_literal: true
+
 require 'sqlite3'
+require 'logger'
+require 'oj'
+require 'yaml'
 
 module Litesupport
 
@@ -90,7 +95,7 @@ module Litesupport
   # common db object options
   def self.create_db(path)
     db = SQLite3::Database.new(path)
-    db.busy_handler{ switch || sleep(0.001) }
+    db.busy_handler{ switch || sleep(0.0001) }
     db.journal_mode = "WAL"
     db.instance_variable_set(:@stmts, {})
     class << db
@@ -167,5 +172,130 @@ module Litesupport
     end
     
   end
+     
+  module ForkListener
+    def self.listeners
+      @listeners ||= []
+    end
+    
+    def self.listen(&block)
+      listeners << block
+    end
+  end
+
+  module Forkable
+      
+    def _fork(*args)
+      ppid = Process.pid
+      result = super
+      if Process.pid != ppid && [:threaded, :iodine].include?(Litesupport.environment)
+        ForkListener.listeners.each{|l| l.call }
+      end
+      result
+    end
+    
+  end
   
+  module Liteconnection
+    
+    include Forkable
+
+    # close, setup, run_stmt and run_sql assume a single connection was created
+    def close
+      @running = false
+      @conn.acquire do |q| 
+        q.stmts.each_pair {|k, v| q.stmts[k].close }
+        q.close
+      end
+    end
+
+    private # all methods are private
+        
+    def init(options = {})
+      #c configure the object, loading options from the appropriate location
+      configure(options)    
+      # setup connections and background threads
+      setup      
+      # handle process exiting
+      at_exit do 
+        exit_callback
+      end
+      # handle forking (restart connections and background threads)
+      Litesupport::ForkListener.listen do
+        setup
+      end
+    end  
+    
+    def configure(options = {})
+      # detect environment (production, development, etc.)
+      env = "development"
+      if defined? Rails
+        env = ENV["RAILS_ENV"]
+      elsif ENV["RACK_ENV"]
+        env = ENV["RACK_ENV"]  
+      elsif ENV["APP_ENV"]
+        env = ENV["RACK_ENV"]
+      end      
+      defaults = self.class::DEFAULT_OPTIONS rescue {}
+      @options = defaults.merge(options)
+      config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
+      config = config[env] if config[env] # if there is a config for the current environment defined then use it, otherwise use the top level declaration
+      config.keys.each do |k| # symbolize keys
+        config[k.to_sym] = config[k]
+        config.delete k
+      end
+      @options.merge!(config)
+      @options.merge!(options) # make sure options passed to initialize trump everything else
+    end
+    
+    def setup
+      @conn = create_pooled_connection
+      @logger = create_logger
+      @running = true
+    end
+    
+    def create_logger
+      @options[:logger] = nil unless @options[:logger]
+      return @options[:logger] if @options[:logger].respond_to? :info
+      return Logger.new(STDOUT) if @options[:logger] == 'STDOUT'
+      return Logger.new(STDERR) if @options[:logger] == 'STDERR'
+      return Logger.new(@options[:logger]) if @options[:logger].is_a? String 
+      return Logger.new(IO::NULL)         
+    end
+    
+    def exit_callback
+      close
+    end
+
+    def run_stmt(stmt, *args)
+      @conn.acquire{|q| q.stmts[stmt].execute!(*args) }
+    end
+
+    def run_sql(sql, *args)
+      @conn.acquire{|q| q.execute(sql, *args) }
+    end
+    
+    def create_pooled_connection(count = 1)
+      Litesupport::Pool.new(1){create_connection}  
+    end
+
+    # common db object options
+    def create_connection
+      conn = SQLite3::Database.new(@options[:path])
+      conn.busy_handler{ Litesupport.switch || sleep(rand * 0.002) }
+      conn.journal_mode = "WAL"
+      conn.synchronous = @options[:sync] || 1
+      conn.mmap_size = @options[:mmap_size] || 0
+      conn.instance_variable_set(:@stmts, {})
+      class << conn
+        attr_reader :stmts
+      end
+      conn
+    end
+    
+  end
+      
 end  
+
+Process.singleton_class.prepend(Litesupport::Forkable)
+

data/lib/litestack/metrics_app.rb

@@ -0,0 +1,5 @@
+require_relative '../lib/litestack/litemetrics'
+
+metric = Litemetrics.instance
+
+puts metric.ids

data/lib/litestack/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litestack
-  VERSION = "0.1.8"
+  VERSION = "0.2.1"
 end       

data/lib/litestack.rb

@@ -1,15 +1,24 @@
 # frozen_string_literal: true
 
 # load core classes
-#require_relative "./version"
-require_relative "litestack/litesupport"
-#require_relative "litedb"
-require_relative "litestack/litecache"
-require_relative "litestack/litejob"
+require_relative "./litestack/version"
+require_relative "./litestack/litesupport"
+require_relative "./litestack/litemetric"
+require_relative "./litestack/litedb"
+require_relative "./litestack/litecache"
+require_relative "./litestack/litejob"
+require_relative "./litestack/litecable"
 
 # conditionally load integration with other libraries
-#require_relative "../sequel/adapters/litedb" if defined? Sequel
-#require_relative "../active_record/connection_adapters/litedb_adapter" if defined? ActiveRecord
-require_relative "active_support/cache/litecache" if defined? ActiveSupport
-require_relative "active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
-#require_relative "../railties/rails/commands/dbconsole" if defined? Rails
+require_relative "./sequel/adapters/litedb" if defined? Sequel
+require_relative "./active_record/connection_adapters/litedb_adapter" if defined? ActiveRecord
+require_relative "./railties/rails/commands/dbconsole" if defined? Rails && defined? ActiveRecord
+require_relative "./active_support/cache/litecache" if defined? ActiveSupport
+require_relative "./active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
+require_relative "./action_cable/subscription_adapter/litecable" if defined? ActionCable
+
+module Litestack
+  class NotImplementedError < Exception; end
+  class TimeoutError < Exception; end
+  class DeadlockError < Exception; end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: litestack
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.1
 platform: ruby
 authors:
 - Mohamed Hassan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-08 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -52,6 +52,7 @@ files:
 - README.md
 - Rakefile
 - WHYLITESTACK.md
+- assets/litecable_logo_teal.png
 - assets/litecache_logo_teal.png
 - assets/litedb_logo_teal.png
 - assets/litejob_logo_teal.png
@@ -63,22 +64,28 @@ files:
 - bench/bench_jobs_rails.rb
 - bench/bench_jobs_raw.rb
 - bench/bench_queue.rb
-- bench/bench_rails.rb
-- bench/bench_raw.rb
 - bench/rails_job.rb
 - bench/skjob.rb
 - bench/uljob.rb
+- lib/action_cable/subscription_adapter/litecable.rb
 - lib/active_job/queue_adapters/litejob_adapter.rb
-- lib/active_job/queue_adapters/ultralite_adapter.rb
 - lib/active_record/connection_adapters/litedb_adapter.rb
 - lib/active_support/cache/litecache.rb
 - lib/litestack.rb
+- lib/litestack/litecable.rb
+- lib/litestack/litecable.sql.yml
 - lib/litestack/litecache.rb
+- lib/litestack/litecache.sql.yml
+- lib/litestack/litecache.yml
 - lib/litestack/litedb.rb
 - lib/litestack/litejob.rb
 - lib/litestack/litejobqueue.rb
+- lib/litestack/litemetric.rb
+- lib/litestack/litemetric.sql.yml
 - lib/litestack/litequeue.rb
+- lib/litestack/litequeue.sql.yml
 - lib/litestack/litesupport.rb
+- lib/litestack/metrics_app.rb
 - lib/litestack/version.rb
 - lib/railties/rails/commands/dbconsole.rb
 - lib/sequel/adapters/litedb.rb
@@ -108,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.8
 signing_key: 
 specification_version: 4
 summary: A SQLite based, lightning fast, super efficient and dead simple to setup

data/bench/bench_rails.rb

@@ -1,81 +0,0 @@
-require 'ultralite'
-require 'active_support'
-require './bench'
-
-cache = ActiveSupport::Cache.lookup_store(:ultralite_cache_store, {})
-mem = ActiveSupport::Cache.lookup_store(:ultralite_cache_store, {path: ":memory:"})
-redis = ActiveSupport::Cache.lookup_store(:redis_cache_store, {})
-
-values = []
-keys = []
-count = 1000
-
-[10, 100, 1000, 10000].each do |size|
-  count.times do
-    keys << random_str(10) 
-    values << random_str(size) 
-  end
-  
-  random_keys = keys.shuffle
-  puts "Benchmarks for values of size #{size} bytes"
-  puts "=========================================================="
-  puts "== Writes =="
-  bench("Ultralite cache writes", count) do |i|
-    cache.write(keys[i], values[i])
-  end
-
-  bench("Ultralite memory cache writes", count) do |i|
-    mem.write(keys[i], values[i])
-  end
-
-  bench("Redis writes", count) do |i|
-    redis.write(keys[i], values[i])
-  end
-
-  puts "== Reads =="
-  bench("Ultralite cache reads", count) do |i|
-    cache.read(random_keys[i])
-  end
-
-  bench("Ultralite memory cache reads", count) do |i|
-    mem.read(random_keys[i])
-  end
-
-  bench("Redis reads", count) do |i|
-    redis.read(random_keys[i])
-  end
-  puts "=========================================================="
-
-
-  keys = []
-  values = []
-end
-
-
-cache.write("somekey", 1, raw: true)
-#puts cache.read("somekey", raw: true)
-
-mem.write("somekey", 1, raw: true)
-#puts mem.read("somekey", raw: true)
-
-redis.write("somekey", 1, raw: true)
-#puts redis.read("somekey", raw: true)
-
-puts "Benchmarks for incrementing integer values"
-puts "=========================================================="
-
-bench("Ultralite cache increment", count) do
-  cache.increment("somekey", 1, raw: true)
-end
-
-bench("Ultralite memory cache increment", count) do
-  mem.increment("somekey", 1, raw: true)
-end
-
-bench("Redis increment", count) do
-  redis.increment("somekey", 1, raw: true )
-end
-
-cache.clear
-redis.clear
-

data/bench/bench_raw.rb

@@ -1,72 +0,0 @@
-require 'ultralite'
-require './bench'
-require 'redis'
-require 'sqlite3'
-
-cache = Ultralite::Cache.new # default settings
-#mem = Ultralite::Cache.new(path: ":memory:") # default settings
-redis = Redis.new # default settings
-
-values = []
-keys = []
-count = 1000
-count.times { keys << random_str(10) }
-
-[10, 100, 1000, 10000].each do |size|
-  count.times do
-    values << random_str(size) 
-  end
-  
-  random_keys = keys.shuffle
-  puts "Benchmarks for values of size #{size} bytes"
-  puts "=========================================================="
-  puts "== Writes =="
-  bench("Ultralite cache writes", count) do |i|
-    cache.set(keys[i], values[i])
-  end
-
- # bench("Ultralite memory cache writes", count) do |i|
- #   mem.set(keys[i], values[i])
- # end
-
-  bench("Redis writes", count) do |i|
-    redis.set(keys[i], values[i])
-  end
-
-  puts "== Reads =="
-  bench("Ultralite cache reads", count) do |i|
-    cache.get(random_keys[i])
-  end
-
- # bench("Ultralite memory cache reads", count) do |i|
- #   cache.get(random_keys[i])
- # end
-
-  bench("Redis reads", count) do |i|
-    redis.get(random_keys[i])
-  end
-  puts "=========================================================="
-
-  values = []
-end
-
-
-cache.set("somekey", 1)
-#mem.set("somekey", 1)
-redis.set("somekey", 1)
-
-bench("Ultralite cache increment") do
-  cache.increment("somekey", 1)
-end
-
-#bench("Ultralite memory cache increment") do
-#  mem.increment("somekey", 1)
-#end
-
-bench("Redis increment") do
-  redis.incr("somekey")
-end
-
-cache.clear
-redis.flushdb
-

data/lib/active_job/queue_adapters/ultralite_adapter.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../ultralite/job.rb'
-require "active_support/core_ext/enumerable"
-require "active_support/core_ext/array/access"
-require "active_job"
-
-module ActiveJob
-  module QueueAdapters
-    # == Ultralite adapter for Active Job
-    #
-    #
-    #   Rails.application.config.active_job.queue_adapter = :ultralite
-    class UltraliteAdapter
-      
-      DEFAULT_OPTIONS = {
-        config_path: "./config/ultrajob.yml",
-        path: "../db/queue.db",
-        queues: [["default", 1, "spawn"]],
-        workers: 1
-      }      
-    
-      DEFAULT_CONFIG_PATH = "./config/ultrajob.yml"
-
-      def initialize(options={})
-        Job.options = DEFAULT_OPTIONS.merge(options)
-      end
-    
-      def enqueue(job) # :nodoc:
-        Job.queue = job.queue_name
-        Job.perform_async(job.serialize)
-      end
-
-      def enqueue_at(job, timestamp) # :nodoc:
-        Job.queue = job.queue_name
-        Job.perform_at(timestamp, job.serialize)
-      end
-
-      class Job # :nodoc:
-        
-        include ::Ultralite::Job
-  
-        def perform(job_data)
-          Base.execute job_data
-        end
-      end
-    end
-  end
-end