litestack 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f959b5e414b4190205ea516cf7acacf180714fd08f4fca9b90a3c84c2f9d594
-  data.tar.gz: 8350f9398d62a481fb656452cd3e4037237c83f292042f1b3db1e56445801a19
+  metadata.gz: 88034778fbac75441d1a9ed2e066ad6ba1aa4962eb48629de1d9ba7ae85f3468
+  data.tar.gz: d6ac66ffd1e78856bd74aa1d88163b95581494aaa481de88b673d84b5a8b8ff6
 SHA512:
-  metadata.gz: 71c6fbd7f51fca6f8a55e86738fc5f28ae1a8452fe3a1a1f1b9cb6944850b71341c728f1eceb40a72b47a8e9f5ca38d07a83db7a7dbc740275bc4df335001032
-  data.tar.gz: 4fd49dd8559328728c9be9dbcf2740c9da22606501822877fccb16454b67aa0e7e5ecdc35a870752d9f02557c3e6d3397df6c68005a20543dd7b6dfd09873cda
+  metadata.gz: 37043f1eab519ea41e81e5b9cb6b20798ea5f6e7ed7ce99d8105454067129e7f49f4b8357b0855948e6df51bf6d466966802085b67943b1f9416ee90f443502a
+  data.tar.gz: 602d0e03d53eeb8e780c31b2f9bb1b3add550035c3cb3fd07f1d14fe94d94aea50a93dea19ab819454465c7824249bb40f7768ec4eebf6662af713218175071d

data/BENCHMARKS.md

@@ -16,7 +16,7 @@ This produces
 SELECT * FROM posts WHERE id = ?
 ```
 
-|Prcoess Count|ActiveRecord:PostgreSQL|ActiveRecord:litedb|Sequel:PostgreSQL|Sequel:litedb|
+|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
 |-:|-:|-:|-:|-:|
 |1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
 |2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
@@ -33,7 +33,7 @@ This produces
 SELECT * FROM posts WHERE user_id = ? LIMIT 5
 ```
 
-|Prcoess Count|ActiveRecord:PostgreSQL|ActiveRecord:litedb|Sequel:PostgreSQL|Sequel:litedb|
+|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
 |-:|-:|-:|-:|-:|
 |1|345 q/s|482 q/s|937 q/s|1.1K q/s|
 |2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
@@ -51,14 +51,14 @@ This produces
 Update posts SET updated_at = ? WHERE id = ?
 ```
 
-|Prcoess Count|ActiveRecord:PostgreSQL|ActiveRecord:litedb|Sequel:PostgreSQL|Sequel:litedb|
+|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
 |-:|-:|-:|-:|-:|
 |1|125 q/s|484 q/s|129 q/s|2.1K q/s|
 |2|265 q/s|576 q/s|333 q/s|2.5K q/s|
 |4|481 q/s|693 q/s|704 q/s|2.3K q/s|
 |8|898 q/s|748 q/s|1.2K q/s|2.4K q/s|
 
-It is clear the Litedb enjoys a clear advantage for reads and is very competitive for updates until many processes are relentlessly trying to write at the same time non stop.
+It is clear the Litedb enjoys a significant advantage for reads and is very competitive for updates until many processes are relentlessly trying to write at the same time non stop.
 For most applications, even with higher level of concurrency, Litedb will scale super well for reads and provide a very good baseline for writes.
 
 > ![litecache](https://github.com/oldmoe/litestack/blob/master/assets/litecache_logo_teal.png?raw=true)

data/lib/litestack/litecache.rb

@@ -58,21 +58,23 @@ class Litecache
   def initialize(options = {})
     @options = DEFAULT_OPTIONS.merge(options)
     @options[:size] = @options[:min_size] if @options[:size] < @options[:min_size]
-    @cache = create_store
-    @stmts = {
-      :pruner => @cache.prepare("DELETE FROM data WHERE expires_in <= $1"),
-      :extra_pruner => @cache.prepare("DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))"),
-      :limited_pruner => @cache.prepare("DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)"),
-      :toucher => @cache.prepare("UPDATE data SET  last_used = unixepoch('now') WHERE id = $1"),
-      :setter => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in"),
-      :inserter => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')"),
-      :finder => @cache.prepare("SELECT id FROM data WHERE id = $1"),
-      :getter => @cache.prepare("SELECT id, value, expires_in FROM data WHERE id = $1"),
-      :deleter => @cache.prepare("delete FROM data WHERE id = $1 returning value"),
-      :incrementer => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in"),
-      :counter => @cache.prepare("SELECT count(*) FROM data"),
-      :sizer => @cache.prepare("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count") 
+    @sql = {
+      :pruner => "DELETE FROM data WHERE expires_in <= $1",
+      :extra_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))",
+      :limited_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)",
+      :toucher => "UPDATE data SET  last_used = unixepoch('now') WHERE id = $1",
+      :setter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in",
+      :inserter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')",
+      :finder => "SELECT id FROM data WHERE id = $1",
+      :getter => "SELECT id, value, expires_in FROM data WHERE id = $1",
+      :deleter => "delete FROM data WHERE id = $1 returning value",
+      :incrementer => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in",
+      :counter => "SELECT count(*) FROM data",
+      :sizer => "SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count" 
     }
+    @cache = create_store
+    @stmts = {}
+    @sql.each_pair{|k, v| @stmts[k] = @cache.prepare(v)}
     @stats = {hit: 0, miss: 0}
     @last_visited = {}
     @running = true
@@ -83,14 +85,12 @@ class Litecache
   def set(key, value, expires_in = nil)
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
-    Litesupport.synchronize do
-      begin
-        @stmts[:setter].execute!(key, value, expires_in)
-      rescue SQLite3::FullException
-        @stmts[:extra_pruner].execute!(0.2)
-        @cache.execute("vacuum")
-        retry
-      end
+    begin
+      @stmts[:setter].execute!(key, value, expires_in)
+    rescue SQLite3::FullException
+      @stmts[:extra_pruner].execute!(0.2)
+      @cache.execute("vacuum")
+      retry
     end
     return true
   end
@@ -99,17 +99,15 @@ class Litecache
   def set_unless_exists(key, value, expires_in = nil)
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
-    Litesupport.synchronize do
-      begin
-        transaction(:immediate) do
-          @stmts[:inserter].execute!(key, value, expires_in)
-          changes = @cache.changes
-        end
-      rescue SQLite3::FullException
-        @stmts[:extra_pruner].execute!(0.2)
-        @cache.execute("vacuum")
-        retry
+    begin
+      transaction(:immediate) do
+        @stmts[:inserter].execute!(key, value, expires_in)
+        changes = @cache.changes
       end
+    rescue SQLite3::FullException
+      @stmts[:extra_pruner].execute!(0.2)
+      @cache.execute("vacuum")
+      retry
     end
     return changes > 0
   end
@@ -118,11 +116,7 @@ class Litecache
   # if the key doesn't exist or it is expired then null will be returned
   def get(key)
     key = key.to_s
-    record = nil
-    Litesupport.synchronize do
-      record = @stmts[:getter].execute!(key)[0]
-    end
-    if record
+    if record = @stmts[:getter].execute!(key)[0]
       @last_visited[key] = true
       @stats[:hit] +=1
       return record[1]
@@ -133,18 +127,14 @@ class Litecache
   
   # delete a key, value pair from the cache
   def delete(key)
-    Litesupport.synchronize do
-      @stmts[:deleter].execute!(key)
-      return @cache.changes > 0
-    end
+    @stmts[:deleter].execute!(key)
+    return @cache.changes > 0
   end
   
   # increment an integer value by amount, optionally add an expiry value (in seconds)
   def increment(key, amount, expires_in = nil)
     expires_in = @expires_in unless expires_in
-    Litesupport.synchronize do
-      @stmts[:incrementer].execute!(key.to_s, amount, expires_in)
-    end
+    @stmts[:incrementer].execute!(key.to_s, amount, expires_in)
   end
   
   # decrement an integer value by amount, optionally add an expiry value (in seconds)
@@ -154,43 +144,35 @@ class Litecache
   
   # delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
   def prune(limit=nil)
-    Litesupport.synchronize do
-      if limit and limit.is_a? Integer
-        @stmts[:limited_pruner].execute!(limit)
-      elsif limit and limit.is_a? Float
-        @stmts[:extra_pruner].execute!(limit)
-      else
-        @stmts[:pruner].execute!      
-      end
+    if limit and limit.is_a? Integer
+      @stmts[:limited_pruner].execute!(limit)
+    elsif limit and limit.is_a? Float
+      @stmts[:extra_pruner].execute!(limit)
+    else
+      @stmts[:pruner].execute!      
     end
   end
   
   # return the number of key, value pairs in the cache
   def count
-    Litesupport.synchronize do
-      @stmts[:counter].execute!.to_a[0][0]
-    end
+    @stmts[:counter].execute!.to_a[0][0]
   end
   
   # return the actual size of the cache file
   def size
-    Litesupport.synchronize do
-      @stmts[:sizer].execute!.to_a[0][0]
-    end
+    @stmts[:sizer].execute!.to_a[0][0]
   end
   
   # delete all key, value pairs in the cache
   def clear
-    Litesupport.synchronize do  
-      @cache.execute("delete FROM data")
-    end
+    @cache.execute("delete FROM data")
   end
   
   # close the connection to the cache file
   def close
     @running = false
     #Litesupport.synchronize do
-      #@cache.close
+    @cache.close
     #end
   end
   
@@ -219,21 +201,31 @@ class Litecache
   
   def spawn_worker
     Litesupport.spawn do
+      # create a specific cache instance for this worker
+      # to overcome SQLite3 Database is locked error
+      cache = create_store
+      stmts = {}
+      [:toucher, :pruner, :extra_pruner].each do |stmt|
+        stmts[stmt] = cache.prepare(@sql[stmt])
+      end
       while @running
         Litesupport.synchronize do
           begin
-            @cache.transaction(:immediate) do
-              @last_visited.delete_if do |k|
-                @stmts[:toucher].execute!(k) || true
+            cache.transaction(:immediate) do
+              @last_visited.delete_if do |k| # there is a race condition here, but not a serious one
+                stmts[:toucher].execute!(k) || true
               end
-              @stmts[:pruner].execute!             
+              stmts[:pruner].execute!             
             end
+          rescue SQLite3::BusyException
+            retry
           rescue SQLite3::FullException
-            @stmts[:extra_pruner].execute!(0.2)
+            stmts[:extra_pruner].execute!(0.2)
           end
         end
         sleep @options[:sleep_interval]
       end
+      cache.close
     end
   end
   

data/lib/litestack/litejob.rb

@@ -43,21 +43,21 @@ module Litejob
   private
   def self.included(klass)
     klass.extend(ClassMethods)
-    klass.get_queue
+    klass.get_jobqueue
   end
   
   module ClassMethods
     def perform_async(*params)
-      get_queue.push(self.name, params, 0, queue)      
+      get_jobqueue.push(self.name, params, 0, queue)      
     end
     
     def perform_at(time, *params)
       delay = time - Time.now.to_i
-      get_queue.push(self.name, params, delay, queue)           
+      get_jobqueue.push(self.name, params, delay, queue)           
     end
     
     def perfrom_in(delay, *params)
-      get_queue.push(self.name, params, delay, queue)           
+      get_jobqueue.push(self.name, params, delay, queue)           
     end
         
     def options
@@ -76,8 +76,8 @@ module Litejob
       @@queue_name = queue_name.to_s
     end
           
-    def get_queue
-      Litejobqueue.queue(options)
+    def get_jobqueue
+      Litejobqueue.jobqueue(options)
     end
   end
     

data/lib/litestack/litejobqueue.rb

@@ -41,7 +41,7 @@ class Litejobqueue
   
   # a method that returns a single instance of the job queue
   # for use by Litejob
-  def self.queue(options = {})
+  def self.jobqueue(options = {})
     @@queue ||= Litesupport.synchronize{self.new(options)}
   end
 
@@ -58,8 +58,8 @@ class Litejobqueue
   def initialize(options = {})
     @options = DEFAULT_OPTIONS.merge(options)
     @worker_sleep_index = 0
-    config = YAML.load_file(@options[:config_path]) rescue {} #an empty hash won't hurt
-    config.each_key do |k| # symbolize keys
+    config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
+    config.keys.each do |k| # symbolize keys
       config[k.to_sym] = config[k]
       config.delete k
     end
@@ -116,14 +116,16 @@ class Litejobqueue
   # create a worker according to environment
   def create_worker
     Litesupport.spawn do
-      Litesupport.switch
+      # we create a queue object specific to this worker here
+      # this way we can survive potential SQLite3 Database is locked errors
+      queue = Litequeue.new(@options)
       loop do
         processed = 0
         @queues.each do |level| # iterate through the levels
           level[1].each do |q| # iterate through the queues in the level
             index = 0
             max = level[0]
-            while index < max && payload = @queue.pop(q[0])  
+            while index < max && payload = queue.pop(q[0])  
               processed += 1
               index += 1
               begin

data/lib/litestack/litequeue.rb

@@ -50,11 +50,7 @@ class Litequeue
   
   # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
   def pop(queue='default')
-    result = nil
-    Litesupport.synchronize do
-      result = @pop.execute!(queue)[0]
-    end
-    result 
+    @pop.execute!(queue)[0]
   end
   
   # delete an item from the queue

data/lib/litestack/litesupport.rb

@@ -37,11 +37,14 @@ module Litesupport
   def self.switch
     if self.environment == :fiber
       Fiber.scheduler.yield
+      true
     elsif self.environment == :polyphony
       Fiber.current.schedule
       Thread.current.switch_fiber
+      true
     else
       # do nothing in case of thread, switching will auto-happen
+      false
     end   
   end
   
@@ -66,7 +69,7 @@ module Litesupport
   # common db object options
   def self.create_db(path)
     db = SQLite3::Database.new(path)
-    db.busy_handler{ sleep 0.001 }
+    db.busy_handler{ switch || sleep(0.001) }
     db.journal_mode = "WAL"
     db
   end

data/lib/litestack/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litestack
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end       

data/lib/sequel/adapters/litedb.rb

@@ -1,6 +1,7 @@
 require_relative '../../litestack/litedb'
 require 'sequel'
 require 'sequel/adapters/sqlite'
+#require 'shared/litedb'
 
 module Sequel
   module Litedb
@@ -12,15 +13,19 @@ module Sequel
       
       set_adapter_scheme :litedb
       
-      
       def connect(server)
+      
+        Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litesupport.environment
+      
         opts = server_opts(server)
         opts[:database] = ':memory:' if blank_object?(opts[:database])
         sqlite3_opts = {}
         sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
         db = ::Litedb.new(opts[:database].to_s, sqlite3_opts)
+
+        self.transaction_mode = :immediate
         
-	    if sqlite_version >= 104
+  	    if sqlite_version >= 104
           db.extended_result_codes = true
         end
         
@@ -29,8 +34,8 @@ module Sequel
         class << db
           attr_reader :prepared_statements
         end
-        db.instance_variable_set(:@prepared_statements, {})
         
+        db.instance_variable_set(:@prepared_statements, {})
         db
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: litestack
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Mohamed Hassan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-26 00:00:00.000000000 Z
+date: 2023-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3