litestack 0.1.1

data/lib/litestack/litedb.rb

@@ -0,0 +1,47 @@
+# all components should require the support module
+require_relative 'litesupport'
+
+# Litedb inherits from the SQLite3::Database class and adds a few initialization options
+class Litedb < ::SQLite3::Database
+
+  # overrride the original initilaizer to allow for connection configuration
+  def initialize(file, options = {}, zfs = nil )
+    if block_given?
+      super(file, options, zfs) do |db|
+        init unless options[:noinit] == true
+        yield db
+      end
+    else
+      super(file, options, zfs)
+      init unless options[:noinit] == true
+    end
+  end
+
+  # enforce immediate mode to avoid deadlocks for a small performance penalty
+  def transaction(mode = :immediate)
+    super(mode)
+  end
+
+  private
+
+  # default connection configuration values 
+  def init
+    # version 3.37 is required for strict typing support and the newest json operators
+    raise Litesupport::Error if SQLite3::SQLITE_VERSION_NUMBER < 3037000
+    # time to wait to obtain a write lock before raising an exception
+    self.busy_handler{|i| sleep 0.001}
+    # level of database durability, 2 = "FULL" (sync on every write), other values include 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
+    self.synchronous = 1
+    # Journal mode WAL allows for greater concurrency (many readers + one writer)
+    self.journal_mode = "WAL"
+    # impose a limit on the WAL file to prevent unlimited growth (with a negative impact on read performance as well)
+    self.journal_size_limit = 64 * 1024 * 1024
+    # set the global memory map so all processes can share data
+    self.mmap_size = 128 * 1024 * 1024 
+    # increase the local connection cache to 2000 pages
+    self.cache_size = 2000
+  end
+
+end
+
+

data/lib/litestack/litejob.rb

@@ -0,0 +1,84 @@
+# frozen_stringe_literal: true
+
+require_relative './litejobqueue'
+
+##
+#Litejob is a Ruby module that enables seamless integration of the Litejobqueue job queueing system into Ruby applications. By including the Litejob module in a class and implementing the #perform method, developers can easily enqueue and process jobs asynchronously.
+#
+#When a job is enqueued, Litejob creates a new instance of the class and passes it any necessary arguments. The class's #perform method is then called asynchronously to process the job. This allows the application to continue running without waiting for the job to finish, improving overall performance and responsiveness.
+#
+#One of the main benefits of using Litejob is its simplicity. Because it integrates directly with Litejobqueue, developers do not need to worry about managing job queues or processing logic themselves. Instead, they can focus on implementing the #perform method to handle the specific job tasks.
+#
+#Litejob also provides a number of useful features, including the ability to set job priorities, retry failed jobs, and limit the number of retries. These features can be configured using simple configuration options in the class that includes the Litejob module.
+#
+#Overall, Litejob is a powerful and flexible module that allows developers to easily integrate Litejobqueue job queueing into their Ruby applications. By enabling asynchronous job processing, Litejob can help improve application performance and scalability, while simplifying the development and management of background job processing logic.
+#  class EasyJob
+#    include ::Litejob
+#   
+#    def perform(params)
+#      # do stuff
+#    end
+#  end
+#
+#Then later you can perform a job asynchronously:
+#
+#  EasyJob.perform_async(params) # perform a job synchronously
+#Or perform it at a specific time:
+#  EasyJob.perform_at(time, params) # perform a job at a specific time
+#Or perform it after a certain delay:
+#  EasyJob.perform_in(delay, params) # perform a job after a certain delay
+#You can also specify a specific queue to be used
+#  class EasyJob
+#    include ::Litejob
+#    
+#    self.queue = :urgent
+#
+#    def perform(params)
+#      # do stuff
+#    end
+#  end
+#
+module Litejob
+
+  private
+  def self.included(klass)
+    klass.extend(ClassMethods)
+    klass.get_queue
+  end
+  
+  module ClassMethods
+    def perform_async(*params)
+      get_queue.push(self.name, params, 0, queue)      
+    end
+    
+    def perform_at(time, *params)
+      delay = time - Time.now.to_i
+      get_queue.push(self.name, params, delay, queue)           
+    end
+    
+    def perfrom_in(delay, *params)
+      get_queue.push(self.name, params, delay, queue)           
+    end
+        
+    def options
+      @@options ||= {}
+    end
+    
+    def options=(options)
+      @@options = options
+    end
+    
+    def queue
+      @@queue_name ||= "default"
+    end
+    
+    def queue=(queue_name)
+      @@queue_name = queue_name.to_s
+    end
+          
+    def get_queue
+      Litejobqueue.queue(options)
+    end
+  end
+    
+end  

data/lib/litestack/litejobqueue.rb

@@ -0,0 +1,161 @@
+# frozen_stringe_literal: true
+
+require 'oj'
+require 'yaml'
+require_relative './litequeue'
+
+##
+#Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
+#
+#One of the main benefits of Litejobqueue is that it is very low on resources, making it an ideal choice for applications that need to manage a large number of jobs without incurring #high resource costs. In addition, because it is built on SQLite, it is easy to use and does not require any additional configuration or setup.
+#
+#Litejobqueue also integrates well with various I/O frameworks like Async and Polyphony, making it a great choice for Ruby applications that use these frameworks. It provides a #simple and easy-to-use API for adding jobs to the queue and for processing them.
+#
+#Overall, LiteJobQueue is an excellent choice for Ruby applications that require a lightweight, embedded job queueing and processing system that is fast, efficient, and easy to use.
+class Litejobqueue
+
+  # the default options for the job queue
+  # can be overriden by passing new options in a hash 
+  # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
+  #   config_path: "./litejob.yml" -> were to find the configuration file (if any)
+  #   path: "./queue.db"
+  #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
+  #   sync: 1 -> sync only when checkpointing
+  #   queues: [["default", 1, "spawn"]] -> an array of queues to process 
+  #   workers: 1 -> number of job processing workers
+  #   sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 3.125] -> sleep intervals for workers
+  # queues will be processed according to priority, such that if the queues are as such
+  #   queues: [["default", 1, "spawn"], ["urgent", 10]]
+  # it means that roughly, if the queues are full, for each 10 urgent jobs, 1 default job will be processed
+  # the priority value is mandatory. The optional "spawn" parameter tells the job workers to spawn a separate execution context (thread or fiber, based on environment) for each job. 
+  # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory. 
+  DEFAULT_OPTIONS = {
+    config_path: "./litejob.yml",
+    path: "./queue.db",
+    queues: [["default", 1, "spawn"]],
+    workers: 1,
+    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 3.125]
+  }
+  
+  @@queue = nil
+  
+  # a method that returns a single instance of the job queue
+  # for use by Litejob
+  def self.queue(options = {})
+    @@queue ||= Litesupport.synchronize{self.new(options)}
+  end
+
+  def self.new(options = {})
+    return @@queue if @@queue
+    @@queue = allocate
+    @@queue.send(:initialize, options)
+    @@queue 
+  end
+
+  # create new queue instance (only once instance will be created in the process)
+  #   jobqueue = Litejobqueue.new
+  #   
+  def initialize(options = {})
+    @options = DEFAULT_OPTIONS.merge(options)
+    @worker_sleep_index = 0
+    config = YAML.load_file(@options[:config_path]) rescue {} #an empty hash won't hurt
+    config.each_key do |k| # symbolize keys
+      config[k.to_sym] = config[k]
+      config.delete k
+    end
+    @options.merge!(config)
+    @queue = Litequeue.new(@options) # create a new queue object
+    # group and order queues according to their priority
+    pgroups = {}
+    @options[:queues].each do |q|
+      pgroups[q[1]] = [] unless pgroups[q[1]]
+      pgroups[q[1]] << [q[0], q[2] == "spawn"]
+    end
+    @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
+    @workers = @options[:workers].times.collect{create_worker} 
+  end
+  
+  # push a job to the queue
+  #   class EasyJob
+  #      def perform(any, number, of_params)
+  #         # do anything
+  #      end 
+  #   end
+  #   jobqueue = Litejobqueue.new
+  #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
+  def push(jobclass, params, delay=0, queue=nil)
+    payload = Oj.dump([jobclass, params])
+    @queue.push(payload, delay, queue)
+  end
+  
+  # delete a job from the job queue
+  #   class EasyJob
+  #      def perform(any, number, of_params)
+  #         # do anything
+  #      end 
+  #   end
+  #   jobqueue = Litejobqueue.new
+  #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
+  #   jobqueue.delete(id)    
+  def delete(id)
+    job = @queue.delete(id)
+    Oj.load(job) if job
+  end
+  
+  private
+  
+  # optionally run a job in its own context
+  def schedule(spawn = false, &block)
+    if spawn
+      Litesupport.spawn &block
+    else
+      yield
+    end
+  end
+  
+  # create a worker according to environment
+  def create_worker
+    Litesupport.spawn do
+      Litesupport.switch
+      loop do
+        processed = 0
+        @queues.each do |level| # iterate through the levels
+          level[1].each do |q| # iterate through the queues in the level
+            index = 0
+            max = level[0]
+            while index < max && payload = @queue.pop(q[0])  
+              processed += 1
+              index += 1
+              begin
+                id, job = payload[0], payload[1]
+                job = Oj.load(job)
+                klass = eval(job[0])
+                schedule(q[1]) do # run the job in a new context
+                  begin
+                    klass.new.perform(*job[1])
+                  rescue Exception => e
+                    puts e
+                    puts e.message
+                    puts e.backtrace
+                  end
+                end
+              rescue Exception => e
+                puts e
+                puts e.message
+                puts e.backtrace
+              end
+              Litesupport.switch #give other context a chance to run here
+            end
+          end
+        end
+        if processed == 0 
+          sleep @options[:sleep_intervals][@worker_sleep_index]      
+          @worker_sleep_index += 1 if @worker_sleep_index < @options[:sleep_intervals].length - 1          
+        else
+          @worker_sleep_index = 0 # reset the index
+        end
+      end
+    end
+  end     
+  
+end

data/lib/litestack/litequeue.rb

@@ -0,0 +1,105 @@
+# frozen_stringe_literal: true
+
+# all components should require the support module
+require_relative 'litesupport'
+
+##
+#Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
+#
+#Litequeue is built on top of SQLite, which makes it very fast and efficient, even when handling large volumes of data. This lightweight and easy-to-use queueing system serves as a good foundation for building more advanced job processing frameworks that require basic queuing capabilities.
+#
+  
+class Litequeue
+
+  # the default options for the queue
+  # can be overriden by passing new options in a hash 
+  # to Litequeue.new
+  #   path: "./queue.db"
+  #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
+  #   sync: 1 -> sync only when checkpointing
+  
+  DEFAULT_OPTIONS = {
+    path: "./queue.db",
+    mmap_size: 32 * 1024,
+    sync: 1
+  }
+
+  # create a new instance of the litequeue object
+  # accepts an optional options hash which will be merged with the DEFAULT_OPTIONS
+  #   queue = Litequeue.new
+  #   queue.push("somevalue", 2) # the value will be ready to pop in 2 seconds
+  #   queue.pop # => nil
+  #   sleep 2
+  #   queue.pop # => "somevalue"
+  
+  def initialize(options = {})
+    @options = DEFAULT_OPTIONS.merge(options)
+    @queue = create_db #(@options[:path])
+    prepare
+  end
+   
+  # push an item to the queue, optionally specifying the queue name (defaults to default) and after how many seconds it should be ready to pop (defaults to zero)
+  # a unique job id is returned from this method, can be used later to delete it before it fires. You can push string, integer, float, true, false or nil values
+  #
+  def push(value, delay=0, queue='default')
+    result = @push.execute!(queue, delay, value)[0]
+    return result[0] if result
+  end
+  
+  alias_method :"<<", :push
+  
+  # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
+  def pop(queue='default')
+    result = nil
+    Litesupport.synchronize do
+      result = @pop.execute!(queue)[0]
+    end
+    result 
+  end
+  
+  # delete an item from the queue
+  #   queue = Litequeue.new
+  #   id = queue.push("somevalue")
+  #   queue.delete(id) # => "somevalue"
+  #   queue.pop # => nil
+  def delete(id, queue='default')
+    fire_at, id = id.split("_")
+    result = @deleter.execute!(queue, fire_at.to_i, id)[0]
+  end
+  
+  # deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
+  def clear(queue=nil)
+    @queue.execute("DELETE FROM _ul_queue_ WHERE iif(?, queue = ?,  1)", queue)
+  end
+
+  # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
+  def count(queue=nil)
+    @queue.get_first_value("SELECT count(*) FROM _ul_queue_ WHERE iif(?, queue = ?, 1)", queue)
+  end
+  
+  # return the size of the queue file on disk
+  def size
+    @queue.get_first_value("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")
+  end
+
+  private  
+  
+  def create_db
+    db = Litesupport.create_db(@options[:path])
+    db.synchronous = @options[:sync]
+    db.wal_autocheckpoint = 10000 
+    db.mmap_size = @options[:mmap_size]
+    db.execute("CREATE TABLE IF NOT EXISTS _ul_queue_(queue TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE, fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, id TEXT DEFAULT(hex(randomblob(8)) || (strftime('%f') * 100)) NOT NULL ON CONFLICT REPLACE, value TEXT, created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE, PRIMARY KEY(queue, fire_at ASC, id) ) WITHOUT ROWID")
+    db
+  end
+
+  def prepare
+     @push = @queue.prepare("INSERT INTO _ul_queue_(queue, fire_at, value) VALUES ($1, (strftime('%s') + $2), $3) RETURNING fire_at || '-' || id")
+     @pop = @queue.prepare("DELETE FROM _ul_queue_ WHERE (queue, fire_at, id) = (SELECT queue, min(fire_at), id FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at <= (unixepoch()) limit 1) RETURNING fire_at || '-' || id, value")
+     @deleter = @queue.prepare("DELETE FROM _ul_queue_ WHERE queue = ifnull($1, 'default') AND fire_at = $2 AND id = $3 RETURNING value")
+  end
+
+end  
+
+
+

data/lib/litestack/litesupport.rb

@@ -0,0 +1,74 @@
+require 'sqlite3'
+
+module Litesupport
+
+  class Error < StandardError; end
+  
+  # cache the environment we are running in
+  # it is an error to change the environment for a process 
+  # or for a child forked from that process
+  def self.environment
+    @env ||= detect_environment
+  end
+  
+  # identify which environment we are running in
+  # we currently support :fiber, :polyphony, :iodine & :threaded
+  # in the future we might want to expand to other environments
+  def self.detect_environment
+    return :fiber if Fiber.scheduler 
+    return :polyphony if defined? Polyphony
+    return :iodine if defined? Iodine
+    return :threaded # fall back for all other environments
+  end
+  
+  # spawn a new execution context
+  def self.spawn(&block)
+    if self.environment == :fiber
+      Fiber.schedule(&block)
+    elsif self.environment == :polyphony
+      spin(&block)
+    elsif self.environment == :threaded or self.environment == :iodine
+      Thread.new(&block)
+    end
+    # we should never reach here
+  end
+  
+  # switch the execution context to allow others to run
+  def self.switch
+    if self.environment == :fiber
+      Fiber.scheduler.yield
+    elsif self.environment == :polyphony
+      Fiber.current.schedule
+      Thread.current.switch_fiber
+    else
+      # do nothing in case of thread, switching will auto-happen
+    end   
+  end
+  
+  # mutex initialization
+  def self.mutex
+    # a single mutex per process (is that ok?)
+    @@mutex ||= Mutex.new
+  end
+  
+  # bold assumption, we will only synchronize threaded code
+  # if some code explicitly wants to synchronize a fiber
+  # they must send (true) as a parameter to this method
+  # else it is a no-op for fibers
+  def self.synchronize(fiber_sync = false, &block)
+    if self.environment == :fiber or self.environment == :polyphony
+      yield # do nothing, just run the block as is
+    else
+      self.mutex.synchronize(&block)
+    end
+  end
+  
+  # common db object options
+  def self.create_db(path)
+    db = SQLite3::Database.new(path)
+    db.busy_handler{ sleep 0.001 }
+    db.journal_mode = "WAL"
+    db
+  end
+  
+end  

data/lib/litestack/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Litestack
+  VERSION = "0.1.1"
+end       

data/lib/litestack.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# load core classes
+#require_relative "./version"
+require_relative "litestack/litesupport"
+#require_relative "litedb"
+require_relative "litestack/litecache"
+require_relative "litestack/litejob"
+
+# conditionally load integration with other libraries
+#require_relative "../sequel/adapters/litedb" if defined? Sequel
+#require_relative "../active_record/connection_adapters/litedb_adapter" if defined? ActiveRecord
+require_relative "active_support/cache/litecache" if defined? ActiveSupport
+require_relative "active_job/queue_adapters/litejob_adapter" if defined? ActiveJob
+#require_relative "../railties/rails/commands/dbconsole" if defined? Rails

data/lib/railties/rails/commands/dbconsole.rb

@@ -0,0 +1,87 @@
+module Rails
+  class DBConsole
+  
+    def start
+      ENV["RAILS_ENV"] ||= @options[:environment] || environment
+      config = db_config.configuration_hash
+
+      case db_config.adapter
+      when /^(jdbc)?mysql/
+        args = {
+          host: "--host",
+          port: "--port",
+          socket: "--socket",
+          username: "--user",
+          encoding: "--default-character-set",
+          sslca: "--ssl-ca",
+          sslcert: "--ssl-cert",
+          sslcapath: "--ssl-capath",
+          sslcipher: "--ssl-cipher",
+          sslkey: "--ssl-key"
+        }.filter_map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }
+
+        if config[:password] && @options[:include_password]
+          args << "--password=#{config[:password]}"
+        elsif config[:password] && !config[:password].to_s.empty?
+          args << "-p"
+        end
+
+        args << db_config.database
+
+        find_cmd_and_exec(["mysql", "mysql5"], *args)
+
+      when /^postgres|^postgis/
+        ENV["PGUSER"]         = config[:username] if config[:username]
+        ENV["PGHOST"]         = config[:host] if config[:host]
+        ENV["PGPORT"]         = config[:port].to_s if config[:port]
+        ENV["PGPASSWORD"]     = config[:password].to_s if config[:password] && @options[:include_password]
+        ENV["PGSSLMODE"]      = config[:sslmode].to_s if config[:sslmode]
+        ENV["PGSSLCERT"]      = config[:sslcert].to_s if config[:sslcert]
+        ENV["PGSSLKEY"]       = config[:sslkey].to_s if config[:sslkey]
+        ENV["PGSSLROOTCERT"]  = config[:sslrootcert].to_s if config[:sslrootcert]
+        find_cmd_and_exec("psql", db_config.database)
+
+      when "sqlite3", "litedb"
+        args = []
+
+        args << "-#{@options[:mode]}" if @options[:mode]
+        args << "-header" if @options[:header]
+        args << File.expand_path(db_config.database, Rails.respond_to?(:root) ? Rails.root : nil)
+
+        find_cmd_and_exec("sqlite3", *args)
+
+
+      when "oracle", "oracle_enhanced"
+        logon = ""
+
+        if config[:username]
+          logon = config[:username].dup
+          logon << "/#{config[:password]}" if config[:password] && @options[:include_password]
+          logon << "@#{db_config.database}" if db_config.database
+        end
+
+        find_cmd_and_exec("sqlplus", logon)
+
+      when "sqlserver"
+        args = []
+
+        args += ["-d", "#{db_config.database}"] if db_config.database
+        args += ["-U", "#{config[:username]}"] if config[:username]
+        args += ["-P", "#{config[:password]}"] if config[:password]
+
+        if config[:host]
+          host_arg = +"tcp:#{config[:host]}"
+          host_arg << ",#{config[:port]}" if config[:port]
+          args += ["-S", host_arg]
+        end
+
+        find_cmd_and_exec("sqlcmd", *args)
+
+      else
+        abort "Unknown command-line client for #{db_config.database}."
+      end
+    end
+                                                                                                                                                                                           
+
+  end
+end

data/lib/sequel/adapters/litedb.rb

@@ -0,0 +1,43 @@
+require_relative '../../litestack/litedb'
+require 'sequel'
+require 'sequel/adapters/sqlite'
+
+module Sequel
+  module Litedb
+  	include SQLite
+
+    LITEDB_TYPES = SQLITE_TYPES
+    
+    class Database < Sequel::SQLite::Database
+      
+      set_adapter_scheme :litedb
+      
+      
+      def connect(server)
+        opts = server_opts(server)
+        opts[:database] = ':memory:' if blank_object?(opts[:database])
+        sqlite3_opts = {}
+        sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
+        db = ::Litedb.new(opts[:database].to_s, sqlite3_opts)
+        
+	    if sqlite_version >= 104
+          db.extended_result_codes = true
+        end
+        
+        connection_pragmas.each{|s| log_connection_yield(s, db){db.execute_batch(s)}}
+        
+        class << db
+          attr_reader :prepared_statements
+        end
+        db.instance_variable_set(:@prepared_statements, {})
+        
+        db
+      end
+
+    end
+    
+    class Dataset < Sequel::SQLite::Dataset      
+    end
+    
+  end
+end

data/samples/ultrajob.yaml

@@ -0,0 +1,2 @@
+queues: 
+  - [normal, 5]