litestack 0.1.1

data/bench/skjob.rb

@@ -0,0 +1,13 @@
+require 'sidekiq'
+
+class SidekiqJob
+  include Sidekiq::Job
+  @@count = 0
+  def perform(count, time)
+    sleep 0.01  
+    @@count += 1
+    if @@count == count  
+      puts "finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+    end
+  end
+end

data/bench/uljob.rb

@@ -0,0 +1,15 @@
+require './bench'
+require '../lib/litestack'
+
+class MyJob
+  include Litejob
+  @@count = 0
+#  self.queue = :normal
+  def perform(count, time)
+    sleep 1
+    @@count += 1
+    if @@count == count  
+      puts "UL finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+    end
+  end
+end

data/lib/active_job/queue_adapters/litejob_adapter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../../litestack/litejob.rb'
+require "active_support/core_ext/enumerable"
+require "active_support/core_ext/array/access"
+require "active_job"
+
+module ActiveJob
+  module QueueAdapters
+    # == Ultralite adapter for Active Job
+    #
+    #
+    #   Rails.application.config.active_job.queue_adapter = :litejob
+    class LitejobAdapter
+      
+      DEFAULT_OPTIONS = {
+        config_path: "./config/litejob.yml",
+        path: "../db/queue.db",
+        queues: [["default", 1, "spawn"]],
+        workers: 1
+      }      
+    
+      def initialize(options={})
+        Job.options = DEFAULT_OPTIONS.merge(options)
+      end
+    
+      def enqueue(job) # :nodoc:
+        Job.queue = job.queue_name
+        Job.perform_async(job.serialize)
+      end
+
+      def enqueue_at(job, timestamp) # :nodoc:
+        Job.queue = job.queue_name
+        Job.perform_at(timestamp, job.serialize)
+      end
+
+      class Job # :nodoc:
+        
+        include ::Litejob
+  
+        def perform(job_data)
+          Base.execute job_data
+        end
+      end
+    end
+  end
+end

data/lib/active_job/queue_adapters/ultralite_adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../../ultralite/job.rb'
+require "active_support/core_ext/enumerable"
+require "active_support/core_ext/array/access"
+require "active_job"
+
+module ActiveJob
+  module QueueAdapters
+    # == Ultralite adapter for Active Job
+    #
+    #
+    #   Rails.application.config.active_job.queue_adapter = :ultralite
+    class UltraliteAdapter
+      
+      DEFAULT_OPTIONS = {
+        config_path: "./config/ultrajob.yml",
+        path: "../db/queue.db",
+        queues: [["default", 1, "spawn"]],
+        workers: 1
+      }      
+    
+      DEFAULT_CONFIG_PATH = "./config/ultrajob.yml"
+
+      def initialize(options={})
+        Job.options = DEFAULT_OPTIONS.merge(options)
+      end
+    
+      def enqueue(job) # :nodoc:
+        Job.queue = job.queue_name
+        Job.perform_async(job.serialize)
+      end
+
+      def enqueue_at(job, timestamp) # :nodoc:
+        Job.queue = job.queue_name
+        Job.perform_at(timestamp, job.serialize)
+      end
+
+      class Job # :nodoc:
+        
+        include ::Ultralite::Job
+  
+        def perform(job_data)
+          Base.execute job_data
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/litedb_adapter.rb

@@ -0,0 +1,102 @@
+require_relative '../../litestack/litedb'
+require 'active_record'
+require 'active_record/connection_adapters/sqlite3_adapter'
+require 'active_record/tasks/sqlite_database_tasks'
+
+module ActiveRecord
+
+	module ConnectionHandling # :nodoc:
+
+		def litedb_connection(config)
+
+			config = config.symbolize_keys
+
+			# Require database.
+			unless config[:database]
+				raise ArgumentError, "No database file specified. Missing argument: database"
+			end
+
+			# Allow database path relative to Rails.root, but only if the database
+			# path is not the special path that tells sqlite to build a database only
+			# in memory.
+			if ":memory:" != config[:database] && !config[:database].to_s.start_with?("file:")
+				config[:database] = File.expand_path(config[:database], Rails.root) if defined?(Rails.root)
+				dirname = File.dirname(config[:database])
+				Dir.mkdir(dirname) unless File.directory?(dirname)
+			end
+
+			db = ::Litedb.new(
+				config[:database].to_s,
+				config.merge(results_as_hash: true)
+			)
+
+			ConnectionAdapters::LitedbAdapter.new(db, logger, nil, config)
+			
+		rescue Errno::ENOENT => error
+			if error.message.include?("No such file or directory")
+				raise ActiveRecord::NoDatabaseError
+			else
+				raise
+			end
+		end
+	end
+
+	module ConnectionAdapters # :nodoc:
+
+		class LitedbAdapter < SQLite3Adapter
+
+		  ADAPTER_NAME = "litedb"
+  
+      class << self
+        
+        def dbconsole(config, options = {})
+          args = []
+
+          args << "-#{options[:mode]}" if options[:mode]
+          args << "-header" if options[:header]
+          args << File.expand_path(config.database, Rails.respond_to?(:root) ? Rails.root : nil)
+
+          find_cmd_and_exec("sqlite3", *args)
+        end
+        		
+      end
+
+		  NATIVE_DATABASE_TYPES = {
+			  primary_key:  "integer PRIMARY KEY NOT NULL",
+			  string:       { name: "text" },
+			  text:         { name: "text" },
+			  integer:      { name: "integer" },
+			  float:        { name: "real" },
+			  decimal:      { name: "real" },
+			  datetime:     { name: "text" },
+			  time:         { name: "integer" },
+			  date:         { name: "text" },
+			  binary:       { name: "blob" },
+			  boolean:      { name: "integer" },
+			  json:         { name: "text" },
+                          unixtime:     { name: "integer" }
+		  }
+		  
+		  private
+		  
+	      def connect
+            @raw_connection = ::Litedb.new(
+              @config[:database].to_s,
+              @config.merge(results_as_hash: true)
+            )
+            configure_connection
+        end
+		end
+
+	end
+	
+  module Tasks # :nodoc:
+    class LitedbDatabaseTasks < SQLiteDatabaseTasks # :nodoc:
+    end
+    
+    module DatabaseTasks
+      register_task(/ultralite/,       "ActiveRecord::Tasks::LitedbDatabaseTasks")
+
+    end
+	end
+end

data/lib/active_support/cache/litecache.rb

@@ -0,0 +1,100 @@
+require "delegate"
+require "active_support/core_ext/enumerable"
+require "active_support/core_ext/array/extract_options"
+require "active_support/core_ext/numeric/time"
+require "active_support/cache"
+require_relative '../../litestack'
+
+
+module ActiveSupport
+  module Cache
+    class Litecache < Store
+
+      prepend Strategy::LocalCache
+		  
+      def self.supports_cache_versioning?
+        true
+      end
+
+      def initialize(options={})
+        super
+        @options[:return_full_record] = true
+      	@cache = ::Litecache.new(@options) # reachout to the outer litecache class
+      end
+
+      def increment(key, amount = 1, options = nil)
+        key = key.to_s
+        options = merged_options(options)
+        @cache.transaction(:immediate) do
+          if value = read(key, options)
+            value = value.to_i + amount
+            write(key, value, options)
+          end
+        end
+      end
+
+      def decrement(key, amount = 1, options = nil)
+        options = merged_options(options)
+        increment(key, -1 * amount, options[:expires_in])
+      end
+      
+      def prune(limit = nil, time = nil)
+        @cache.prune(limit)
+      end
+
+      def cleanup(limit = nil, time = nil)
+        @cache.prune(limit)
+      end 
+
+      def clear()
+      	@cache.clear
+      end
+      
+      def count
+        @cache.count
+      end
+      
+      def size
+        @cache.size
+      end
+      
+      def max_size
+        @cache.max_size
+      end
+
+      def stats
+        @cache.stats
+      end
+
+      private
+
+        # Read an entry from the cache.
+        def read_entry(key, **options)
+          deserialize_entry(@cache.get(key))
+        end
+
+        # Write an entry to the cache.
+        def write_entry(key, entry, **options)
+          write_serialized_entry(key, serialize_entry(entry, **options), **options)
+        end
+
+        def write_serialized_entry(key, payload, **options)
+          expires_in = options[:expires_in].to_i
+          if options[:race_condition_ttl] && expires_in > 0 && !options[:raw]
+            expires_in += 5.minutes
+          end
+          if options[:unless_exist]
+            @cache.set_unless_exists(key, payload, expires_in)
+          else
+            @cache.set(key, payload, expires_in)
+          end
+        end
+
+        # Delete an entry from the cache.
+        def delete_entry(key, **options)
+          return @cache.delete(key)
+        end
+
+    end
+  end
+end

data/lib/active_support/cache/ultralite_cache_store.rb

@@ -0,0 +1,100 @@
+require "delegate"
+require "active_support/core_ext/enumerable"
+require "active_support/core_ext/array/extract_options"
+require "active_support/core_ext/numeric/time"
+require "active_support/cache"
+require_relative '../../ultralite/cache.rb'
+
+
+module ActiveSupport
+  module Cache
+    class UltraliteCacheStore < Store
+
+      #prepend Strategy::LocalCache
+		  
+      def self.supports_cache_versioning?
+        true
+      end
+
+      def initialize(options={})
+        super
+        @options[:return_full_record] = true
+      	@cache = ::Ultralite::Cache.new(@options)
+      end
+
+      def increment(key, amount = 1, options = nil)
+        key = key.to_s
+        options = merged_options(options)
+        @cache.transaction(:immediate) do
+          if value = read(key, options)
+            value = value.to_i + amount
+            write(key, value, options)
+          end
+        end
+      end
+
+      def decrement(key, amount = 1, options = nil)
+        options = merged_options(options)
+        increment(key, -1 * amount, options[:expires_in])
+      end
+      
+      def prune(limit = nil, time = nil)
+        @cache.prune(limit)
+      end
+
+      def cleanup(limit = nil, time = nil)
+        @cache.prune(limit)
+      end 
+
+      def clear()
+      	@cache.clear
+      end
+      
+      def count
+        @cache.count
+      end
+      
+      def size
+        @cache.size
+      end
+      
+      def max_size
+        @cache.max_size
+      end
+
+      def stats
+        @cache.stats
+      end
+
+      private
+
+        # Read an entry from the cache.
+        def read_entry(key, **options)
+          deserialize_entry(@cache.get(key))
+        end
+
+        # Write an entry to the cache.
+        def write_entry(key, entry, **options)
+          write_serialized_entry(key, serialize_entry(entry, **options), **options)
+        end
+
+        def write_serialized_entry(key, payload, **options)
+          expires_in = options[:expires_in].to_i
+          if options[:race_condition_ttl] && expires_in > 0 && !options[:raw]
+            expires_in += 5.minutes
+          end
+          if options[:unless_exist]
+            @cache.set_unless_exists(key, payload, expires_in)
+          else
+            @cache.set(key, payload, expires_in)
+          end
+        end
+
+        # Delete an entry from the cache.
+        def delete_entry(key, **options)
+          return @cache.delete(key)
+        end
+
+    end
+  end
+end

data/lib/litestack/litecache.rb

@@ -0,0 +1,254 @@
+# frozen_stringe_literal: true
+
+# all components should require the support module
+require_relative 'litesupport'
+
+##
+#Litecache is a caching library for Ruby applications that is built on top of SQLite. It is designed to be simple to use, very fast, and feature-rich, providing developers with a reliable and efficient way to cache data.
+#
+#One of the main features of Litecache is automatic key expiry, which allows developers to set an expiration time for each cached item. This ensures that cached data is automatically removed from the cache after a certain amount of time has passed, reducing the risk of stale data being served to users.
+#
+#In addition, Litecache supports LRU (Least Recently Used) removal, which means that if the cache reaches its capacity limit, the least recently used items will be removed first to make room for new items. This ensures that the most frequently accessed data is always available in the cache.
+#
+#Litecache also supports integer value increment/decrement, which allows developers to increment or decrement the value of a cached item in a thread-safe manner. This is useful for implementing counters or other types of numerical data that need to be updated frequently.
+#
+#Overall, Litecache is a powerful and flexible caching library that provides automatic key expiry, LRU removal, and integer value increment/decrement capabilities. Its fast performance and simple API make it an excellent choice for Ruby applications that need a reliable and efficient way to cache data.
+
+class Litecache
+
+  # the default options for the cache
+  # can be overriden by passing new options in a hash 
+  # to Litecache.new
+  #   path: "./cache.db"
+  #   expiry: 60 * 60 * 24 * 30 -> one month default expiry if none is provided
+  #   size: 128 * 1024 * 1024 -> 128MB 
+  #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
+  #   min_size: 32 * 1024 -> 32MB
+  #   return_full_record: false -> only return the payload
+  #   sleep_interval: 1 -> 1 second of sleep between cleanup runs
+
+  DEFAULT_OPTIONS = {
+    path: "./cache.db",
+    expiry: 60 * 60 * 24 * 30, # one month 
+    size: 128 * 1024 * 1024, #128MB 
+    mmap_size: 128 * 1024 * 1024, #128MB
+    min_size: 32 * 1024, #32MB
+    return_full_record: false, #only return the payload
+    sleep_interval: 1 # 1 second
+  }
+  
+  # creates a new instance of Litecache
+  # can optionally receive an options hash which will be merged
+  # with the DEFAULT_OPTIONS (the new hash overrides any matching keys in the default one).
+  #
+  # Example:
+  #   litecache = Litecache.new
+  #
+  #   litecache.set("a", "somevalue")
+  #   litecache.get("a") # =>  "somevalue"
+  #
+  #   litecache.set("b", "othervalue", 1) # expire aftre 1 second
+  #   litecache.get("b") # => "othervalue"
+  #   sleep 2
+  #   litecache.get("b") # => nil
+  #
+  #   litecache.clear # nothing remains in the cache
+  #   litecache.close # optional, you can safely kill the process
+  
+  def initialize(options = {})
+    @options = DEFAULT_OPTIONS.merge(options)
+    @options[:size] = @options[:min_size] if @options[:size] < @options[:min_size]
+    @cache = create_store
+    @stmts = {
+      :pruner => @cache.prepare("DELETE FROM data WHERE expires_in <= $1"),
+      :extra_pruner => @cache.prepare("DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))"),
+      :limited_pruner => @cache.prepare("DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)"),
+      :toucher => @cache.prepare("UPDATE data SET  last_used = unixepoch('now') WHERE id = $1"),
+      :setter => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in"),
+      :inserter => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')"),
+      :finder => @cache.prepare("SELECT id FROM data WHERE id = $1"),
+      :getter => @cache.prepare("SELECT id, value, expires_in FROM data WHERE id = $1"),
+      :deleter => @cache.prepare("delete FROM data WHERE id = $1 returning value"),
+      :incrementer => @cache.prepare("INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in"),
+      :counter => @cache.prepare("SELECT count(*) FROM data"),
+      :sizer => @cache.prepare("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count") 
+    }
+    @stats = {hit: 0, miss: 0}
+    @last_visited = {}
+    @running = true
+    @bgthread = spawn_worker 
+  end
+  
+  # add a key, value pair to the cache, with an optional expiry value (number of seconds)      
+  def set(key, value, expires_in = nil)
+    key = key.to_s
+    expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
+    Litesupport.synchronize do
+      begin
+        @stmts[:setter].execute!(key, value, expires_in)
+      rescue SQLite3::FullException
+        @stmts[:extra_pruner].execute!(0.2)
+        @cache.execute("vacuum")
+        retry
+      end
+    end
+    return true
+  end
+  
+  # add a key, value pair to the cache, but only if the key doesn't exist, with an optional expiry value (number of seconds)      
+  def set_unless_exists(key, value, expires_in = nil)
+    key = key.to_s
+    expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
+    Litesupport.synchronize do
+      begin
+        transaction(:immediate) do
+          @stmts[:inserter].execute!(key, value, expires_in)
+          changes = @cache.changes
+        end
+      rescue SQLite3::FullException
+        @stmts[:extra_pruner].execute!(0.2)
+        @cache.execute("vacuum")
+        retry
+      end
+    end
+    return changes > 0
+  end
+  
+  # get a value by its key
+  # if the key doesn't exist or it is expired then null will be returned
+  def get(key)
+    key = key.to_s
+    record = nil
+    Litesupport.synchronize do
+      record = @stmts[:getter].execute!(key)[0]
+    end
+    if record
+      @last_visited[key] = true
+      @stats[:hit] +=1
+      return record[1]
+    end
+    @stats[:miss] += 1
+    nil
+  end
+  
+  # delete a key, value pair from the cache
+  def delete(key)
+    Litesupport.synchronize do
+      @stmts[:deleter].execute!(key)
+      return @cache.changes > 0
+    end
+  end
+  
+  # increment an integer value by amount, optionally add an expiry value (in seconds)
+  def increment(key, amount, expires_in = nil)
+    expires_in = @expires_in unless expires_in
+    Litesupport.synchronize do
+      @stmts[:incrementer].execute!(key.to_s, amount, expires_in)
+    end
+  end
+  
+  # decrement an integer value by amount, optionally add an expiry value (in seconds)
+  def decrement(key, amount, expires_in = nil)
+    increment(key, -amount, expires_in)
+  end
+  
+  # delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
+  def prune(limit=nil)
+    Litesupport.synchronize do
+      if limit and limit.is_a? Integer
+        @stmts[:limited_pruner].execute!(limit)
+      elsif limit and limit.is_a? Float
+        @stmts[:extra_pruner].execute!(limit)
+      else
+        @stmts[:pruner].execute!      
+      end
+    end
+  end
+  
+  # return the number of key, value pairs in the cache
+  def count
+    Litesupport.synchronize do
+      @stmts[:counter].execute!.to_a[0][0]
+    end
+  end
+  
+  # return the actual size of the cache file
+  def size
+    Litesupport.synchronize do
+      @stmts[:sizer].execute!.to_a[0][0]
+    end
+  end
+  
+  # delete all key, value pairs in the cache
+  def clear
+    Litesupport.synchronize do  
+      @cache.execute("delete FROM data")
+    end
+  end
+  
+  # close the connection to the cache file
+  def close
+    @running = false
+    #Litesupport.synchronize do
+      #@cache.close
+    #end
+  end
+  
+  # return the maximum size of the cache
+  def max_size
+    Litesupport.synchronize do
+      @cache.get_first_value("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")
+    end
+  end
+  
+  # hits and misses for get operations performed over this particular connection (not cache wide)
+  # 
+  #   litecache.stats # => {hit: 543, miss: 31} 
+  def stats
+    @stats
+  end
+
+  # low level access to SQLite transactions, use with caution
+  def transaction(mode)
+    @cache.transaction(mode) do
+      yield
+    end
+  end
+
+  private 
+  
+  def spawn_worker
+    Litesupport.spawn do
+      while @running
+        Litesupport.synchronize do
+          begin
+            @cache.transaction(:immediate) do
+              @last_visited.delete_if do |k|
+                @stmts[:toucher].execute!(k) || true
+              end
+              @stmts[:pruner].execute!             
+            end
+          rescue SQLite3::FullException
+            @stmts[:extra_pruner].execute!(0.2)
+          end
+        end
+        sleep @options[:sleep_interval]
+      end
+    end
+  end
+  
+  def create_store
+    db = Litesupport.create_db(@options[:path])
+    db.synchronous = 0
+    db.cache_size = 2000
+    db.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
+    db.mmap_size = @options[:mmap_size]
+    db.max_page_count = (@options[:size] / db.page_size).to_i
+    db.case_sensitive_like = true
+    db.execute("CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)")
+    db.execute("CREATE index if not exists expiry_index on data (expires_in)")
+    db.execute("CREATE index if not exists last_used_index on data (last_used)")
+    db
+  end
+  
+end