litestack 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01c10017cec21bcabea357aecb8746779060718bb2e6b6e36f250c37d80656f1
-  data.tar.gz: 863b319c7a658f2a19410c614035068f8a2dec72b189698a5a081f595ecf5853
+  metadata.gz: 2662941f303da99554039370e6e53e4c13956c52070e93ad6862742881ca8063
+  data.tar.gz: ccd5583a9e0e7f5c559f8dafdde913bba256b7b5a679fe927cd31805c1289022
 SHA512:
-  metadata.gz: 7bb0fd84ee582eddfe32e48b0373a7545cad8581217ba56d7ac6c984b60b137b86d5196e1bc37486b88d21c28bc5ff62da573a1ba6ce6b49d66ba744ddbcabc6
-  data.tar.gz: 38da99eae6a37222a8689577900504ea1b131f0ec56f67531dff00f5ae4f4c62a29baca21ba2a102435cc334773f8ef4709a361dff6f28de988471eac0937e08
+  metadata.gz: 1a139b3c42cd3f9a327fc7d8f1487dab2e95f2c88d07e063eecc2e438c91236f301a8e3928c9a3b41030fd8c08f4c118c449e56ea4be35b73b00fe2aa240faf8
+  data.tar.gz: 409221a087df5707bd6da477bb9091daf2ee0fc5f8114d71a7666324676bc62e5d77b6edfd1e3e7d29d601985f7936b2e3545e618b1c888b6c7a748544bbafde

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.1.7] - 2022-03-05
+
+- Code cleanup, removal of references to older name
+- Fix for the litedb rake tasks (thanks: netmute)
+- More fixes for the new concurrency model
+- Introduced a logger for the Litejobqueue (doesn't work with Polyphony, fix should come soon)
+
 ## [0.1.6] - 2022-03-03
 
 - Revamped the locking model, more robust, minimal performance hit

data/README.md

@@ -1,9 +1,9 @@
 ![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
 
 
-litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache, a robust job queue, and a simple yet performant full-text search all in a single package.
+litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache and a robust job queue all in a single package.
 
-Compared to conventional approaches that require separate servers and databases, LiteStack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
+Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
 
 You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
 
@@ -83,7 +83,7 @@ adapter: litedb
 litedb offers integration with the Sequel database toolkit and can be configured as follows
 
 ```ruby
-DB = Sequel.conncet("litedb://path_to_db_file")    
+DB = Sequel.connect("litedb://path_to_db_file")
 ```
 
 

data/bench/bench.rb

@@ -2,8 +2,8 @@ require 'sqlite3'
 
 def bench(msg, iterations=1000)
   GC.start
-  GC.compact
-  print "Starting #{iterations} iterations of #{msg} ... "
+  #GC.compact
+  STDERR.puts "Starting #{iterations} iterations of #{msg} ... "
   t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
   iterations.times do |i|
     yield i
@@ -12,7 +12,7 @@ def bench(msg, iterations=1000)
   time = ((t2 - t1)*1000).to_i.to_f / 1000 rescue 0
   ips = ((iterations/(t2-t1))*100).to_i.to_f / 100 rescue "infinity?"
   #{m: msg, t: time, ips: iteratinos/time, i: iterations}
-  puts "finished in #{time} seconds (#{ips} ips)"
+ STDERR.puts " .. finished in #{time} seconds (#{ips} ips)"
 end
 
 @db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation

data/bench/bench_jobs_raw.rb

@@ -1,29 +1,45 @@
-#require 'polyphony'
-require 'async/scheduler'
 require './bench'
 
-Fiber.set_scheduler Async::Scheduler.new
+count = ARGV[0].to_i rescue 1000
+env = ARGV[1] || "t"
+delay = ARGV[2].to_f rescue 0
 
-count = 10000
+# Sidekiq bench
+###############
 require './skjob.rb'
-require './uljob.rb'
-
-puts Litesupport.environment
 
-t = Time.now.to_f
+t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 puts "make sure sidekiq is started with skjob.rb as the job"     
 bench("enqueuing sidekiq jobs", count) do |i|
-  SidekiqJob.perform_async(count, t)
+  SidekiqJob.perform_async(count, t, delay)
 end
 
 puts "Don't forget to check the sidekiq log for processing time conclusion"
 
-t = Time.now.to_f
+# Litejob bench
+###############
+
+if env == "t" # threaded
+  # do nothing 
+elsif env == "a" # async
+  require 'async/scheduler'
+  Fiber.set_scheduler Async::Scheduler.new
+elsif env == "p" # polyphony
+  require 'polyphony'
+end
+
+require './uljob.rb'
+
+STDERR.puts "litejob started in #{Litesupport.environment} environmnet"
+
+t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 bench("enqueuing litejobs", count) do |i|
-  MyJob.perform_async(count, t)
+  MyJob.perform_async(count, t, delay)
 end
 
-Fiber.scheduler.run
+puts "Please wait for the benchmark to finish .."
+
+Fiber.scheduler.run if env == "a"
 
 sleep
 

data/bench/skjob.rb

@@ -3,11 +3,13 @@ require 'sidekiq'
 class SidekiqJob
   include Sidekiq::Job
   @@count = 0
-  def perform(count, time)
-    sleep 0.1  
+  def perform(count, time, sleep_interval = nil)
+    sleep sleep_interval if sleep_interval  
     @@count += 1
     if @@count == count  
-      puts "finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      STDERR.puts "Sidekiq finished in #{now - time} seconds (#{count / (now - time)} jps)"
+      @@count = 0
     end
   end
 end

data/bench/uljob.rb

@@ -4,12 +4,13 @@ require '../lib/litestack'
 class MyJob
   include Litejob
   @@count = 0
-#  self.queue = :normal
-  def perform(count, time)
-    #sleep 0.1
+  #self.queue = :default
+  def perform(count, time, sleep_interval = nil)
+    sleep sleep_interval if sleep_interval  
     @@count += 1
     if @@count == count  
-      puts "UL finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      STDERR.puts "Litejob finished in #{now - time} seconds (#{count / (now - time)} jps)"
     end
   end
 end

data/lib/active_record/connection_adapters/litedb_adapter.rb

@@ -95,7 +95,7 @@ module ActiveRecord
     end
     
     module DatabaseTasks
-      register_task(/ultralite/,       "ActiveRecord::Tasks::LitedbDatabaseTasks")
+      register_task(/litedb/, "ActiveRecord::Tasks::LitedbDatabaseTasks")
 
     end
 	end

data/lib/litestack/litejobqueue.rb

@@ -1,5 +1,5 @@
 # frozen_stringe_literal: true
-
+require 'logger'
 require 'oj'
 require 'yaml'
 require_relative './litequeue'
@@ -32,9 +32,10 @@ class Litejobqueue
   DEFAULT_OPTIONS = {
     config_path: "./litejob.yml",
     path: "./queue.db",
-    queues: [["default", 5]],
-    workers: 1,
-    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 3.125]
+    queues: [["default", 1]],
+    workers: 5,
+    logger: STDOUT,
+    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
   }
   
   @@queue = nil
@@ -57,7 +58,6 @@ class Litejobqueue
   #   
   def initialize(options = {})
     @options = DEFAULT_OPTIONS.merge(options)
-    @worker_sleep_index = 0
     config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
     config.keys.each do |k| # symbolize keys
       config[k.to_sym] = config[k]
@@ -65,6 +65,11 @@ class Litejobqueue
     end
     @options.merge!(config)
     @queue = Litequeue.new(@options) # create a new queue object
+    if @options[:logger].respond_to? :info
+      @logger = @options[:logger] 
+    else
+      @logger = Logger.new(@options[:logger])
+    end
     # group and order queues according to their priority
     pgroups = {}
     @options[:queues].each do |q|
@@ -72,7 +77,7 @@ class Litejobqueue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
-    @workers = @options[:workers].times.collect{create_worker} 
+    @workers = @options[:workers].times.collect{ create_worker } 
   end
   
   # push a job to the queue
@@ -85,7 +90,10 @@ class Litejobqueue
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
     payload = Oj.dump([jobclass, params])
-    @queue.push(payload, delay, queue)
+    #res = 
+    res = @queue.push(payload, delay, queue)
+    @logger.info("[litejob]:[ENQ] id: #{res} class: #{jobclass}")
+    res
   end
   
   # delete a job from the job queue
@@ -99,7 +107,9 @@ class Litejobqueue
   #   jobqueue.delete(id)    
   def delete(id)
     job = @queue.delete(id)
+    @logger.info("[litejob]:[DEL] job: #{job}")
     Oj.load(job) if job
+    job
   end
   
   private
@@ -111,27 +121,36 @@ class Litejobqueue
     else
       yield
     end
-  end
-  
+  end  
+    
   # create a worker according to environment
   def create_worker
     Litesupport.spawn do
+      if @options[:logger].respond_to? :info
+        logger = @options[:logger] 
+      else
+        logger = Logger.new(@options[:logger])
+      end
+      worker_sleep_index = 0
+      i = 0
       loop do
         processed = 0
         @queues.each do |level| # iterate through the levels
           level[1].each do |q| # iterate through the queues in the level
             index = 0
             max = level[0]
-            while index < max && payload = @queue.pop(q[0]) # fearlessly use the same queue object 
+            while index < max && payload = @queue.pop(q[0], 1) # fearlessly use the same queue object 
               processed += 1
               index += 1
               begin
                 id, job = payload[0], payload[1]
                 job = Oj.load(job)
+                logger.info "[litejob]:[DEQ] id: #{id} class: #{job[0]}" 
                 klass = eval(job[0])
                 schedule(q[1]) do # run the job in a new context
                   begin
                     klass.new.perform(*job[1])
+                    logger.info "[litejob]:[END] id: #{id} class: #{job[0]}" 
                   rescue Exception => e
                     puts e
                     puts e.message
@@ -143,18 +162,18 @@ class Litejobqueue
                 puts e.message
                 puts e.backtrace
               end
-              Litesupport.switch #give other context a chance to run here
+              Litesupport.switch #give other contexts a chance to run here
             end
           end
         end
         if processed == 0 
-          sleep @options[:sleep_intervals][@worker_sleep_index]      
-          @worker_sleep_index += 1 if @worker_sleep_index < @options[:sleep_intervals].length - 1          
+          sleep @options[:sleep_intervals][worker_sleep_index]      
+          worker_sleep_index += 1 if worker_sleep_index < @options[:sleep_intervals].length - 1          
         else
-          @worker_sleep_index = 0 # reset the index
+          worker_sleep_index = 0 # reset the index
         end
       end
     end
-  end     
+  end  
   
 end

data/lib/litestack/litequeue.rb

@@ -52,8 +52,10 @@ class Litequeue
   alias_method :"<<", :push
   
   # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
-  def pop(queue='default')
-    @queue.acquire {|q| q.stmts[:pop].execute!(queue)[0] }
+  def pop(queue='default', limit = 1)
+   res = @queue.acquire {|q| res = q.stmts[:pop].execute!(queue, limit)[0] }
+   #return res[0] if res.length == 1
+   #res
   end
   
   # delete an item from the queue
@@ -90,7 +92,7 @@ class Litequeue
     db.mmap_size = @options[:mmap_size]
     db.execute("CREATE TABLE IF NOT EXISTS _ul_queue_(queue TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE, fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, id TEXT DEFAULT(hex(randomblob(8)) || (strftime('%f') * 100)) NOT NULL ON CONFLICT REPLACE, value TEXT, created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, PRIMARY KEY(queue, fire_at ASC, id) ) WITHOUT ROWID")
     db.stmts[:push] = db.prepare("INSERT INTO _ul_queue_(queue, fire_at, value) VALUES ($1, (strftime('%s') + $2), $3) RETURNING fire_at || '-' || id")
-    db.stmts[:pop] = db.prepare("DELETE FROM _ul_queue_ WHERE (queue, fire_at, id) = (SELECT queue, min(fire_at), id FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at <= (unixepoch()) limit 1) RETURNING fire_at || '-' || id, value")
+    db.stmts[:pop] = db.prepare("DELETE FROM _ul_queue_ WHERE (queue, fire_at, id) IN (SELECT queue, fire_at, id FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at <= (unixepoch()) ORDER BY fire_at ASC LIMIT ifnull($2, 1)) RETURNING fire_at || '-' || id, value")
     db.stmts[:delete] = db.prepare("DELETE FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at = $2 AND id = $3 RETURNING value")
     db
   end

data/lib/litestack/litesupport.rb

@@ -1,4 +1,5 @@
 require 'sqlite3'
+require 'hiredis'
 
 module Litesupport
 
@@ -11,6 +12,11 @@ module Litesupport
     @env ||= detect_environment
   end
   
+  def self.max_contexts
+    return 50 if environment == :fiber || environment == :polyphony
+    5    
+  end
+    
   # identify which environment we are running in
   # we currently support :fiber, :polyphony, :iodine & :threaded
   # in the future we might want to expand to other environments
@@ -55,7 +61,7 @@ module Litesupport
       Thread.current.switch_fiber
       true
     else
-      # do nothing in case of thread, switching will auto-happen
+      #Thread.pass
       false
     end   
   end
@@ -124,7 +130,7 @@ module Litesupport
       result = nil
       while !acquired do
         @mutex.synchronize do
-          if resource = @resources.find{|r| r[1] == :free}
+          if resource = @resources.find{|r| r[1] == :free }
             resource[1] = :busy
             begin
               result = yield resource[0]
@@ -133,11 +139,10 @@ module Litesupport
             ensure
               resource[1] = :free
               acquired = true
-              return nil              
             end
           end
         end
-        sleep 0.0001 unless acquired
+        sleep 0.001 unless acquired
       end
       result
     end

data/lib/litestack/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litestack
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end