litestack 0.4.1 → 0.4.3

data/lib/litestack/litecable.rb

@@ -1,18 +1,16 @@
 # frozen_stringe_literal: true
 
 # all components should require the support module
-require_relative 'litesupport'
-require_relative 'litemetric'
+require_relative "litesupport"
+require_relative "litemetric"
 
-require 'base64'
-require 'oj'
+require "base64"
+require "oj"
 
 class Litecable
-
   include Litesupport::Liteconnection
   include Litemetric::Measurable
 
-
   DEFAULT_OPTIONS = {
     config_path: "./litecable.yml",
     path: Litesupport.root.join("cable.sqlite3"),
@@ -22,56 +20,63 @@ class Litecable
     listen_interval: 0.05, # check new messages every 50 milliseconds
     metrics: false
   }
-  
-  def initialize(options = {})  
-    @messages = Litesupport::Pool.new(1){[]}
+
+  def initialize(options = {})
+    @messages = Litesupport::Pool.new(1) { [] }
     init(options)
     collect_metrics if @options[:metrics]
   end
-  
+
   # broadcast a message to a specific channel
-  def broadcast(channel, payload=nil)
+  def broadcast(channel, payload = nil)
     # group meesages and only do broadcast every 10 ms
     # but broadcast locally normally
-    @messages.acquire{|msgs| msgs << [channel.to_s, Oj.dump(payload)]}
+    @messages.acquire { |msgs| msgs << [channel.to_s, Oj.dump(payload)] }
     capture(:broadcast, channel)
-    local_broadcast(channel, payload) 
+    local_broadcast(channel, payload)
   end
-  
+
   # subscribe to a channel, optionally providing a success callback proc
   def subscribe(channel, subscriber, success_callback = nil)
     @subscribers.acquire do |subs|
       subs[channel] = {} unless subs[channel]
       subs[channel][subscriber] = true
     end
+    success_callback&.call
     capture(:subscribe, channel)
   end
-  
+
   # unsubscribe from a channel
   def unsubscribe(channel, subscriber)
-    @subscribers.acquire{|subs| subs[channel].delete(subscriber) rescue nil }
+    @subscribers.acquire { |subs|
+      begin
+        subs[channel].delete(subscriber)
+      rescue
+        nil
+      end
+    }
     capture(:unsubscribe, channel)
   end
 
-  private 
-  
-  # broadcast the message to local subscribers  
-  def local_broadcast(channel, payload=nil)
+  private
+
+  # broadcast the message to local subscribers
+  def local_broadcast(channel, payload = nil)
     subscribers = []
-    @subscribers.acquire do |subs| 
-      return unless subs[channel]
+    @subscribers.acquire do |subs|
+      break unless subs[channel]
       subscribers = subs[channel].keys
     end
     subscribers.each do |subscriber|
-      subscriber.call(payload)      
+      subscriber.call(payload)
       capture(:message, channel)
     end
-  end  
-      
+  end
+
   def setup
     super # create connection
     @pid = Process.pid
-    @subscribers = Litesupport::Pool.new(1){{}}
+    @subscribers = Litesupport::Pool.new(1) { {} }
     @running = true
     @listener = create_listener
     @pruner = create_pruner
@@ -81,48 +86,47 @@ class Litecable
 
   def create_broadcaster
     Litescheduler.spawn do
-      while @running do
+      while @running
         @messages.acquire do |msgs|
           if msgs.length > 0
             run_sql("BEGIN IMMEDIATE")
-            while msg = msgs.shift
+            while (msg = msgs.shift)
               run_stmt(:publish, msg[0], msg[1], @pid)
             end
             run_sql("END")
-          end          
+          end
         end
         sleep 0.02
-      end      
+      end
     end
   end
 
   def create_pruner
     Litescheduler.spawn do
-      while @running do
+      while @running
         run_stmt(:prune, @options[:expire_after])
         sleep @options[:expire_after]
-      end      
+      end
     end
   end
 
   def create_listener
     Litescheduler.spawn do
-      while @running do
+      while @running
         @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
         run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
           @logger.info "RECEIVED #{msg}"
           @last_fetched_id = msg[0]
-          local_broadcast(msg[1], Oj.load(msg[2])) 
+          local_broadcast(msg[1], Oj.load(msg[2]))
         end
         sleep @options[:listen_interval]
-      end      
+      end
     end
   end
 
   def create_connection
     super("#{__dir__}/litecable.sql.yml") do |conn|
-      conn.wal_autocheckpoint = 10000 
+      conn.wal_autocheckpoint = 10000
     end
   end
-  
 end

data/lib/litestack/litecable.sql.yml

@@ -2,23 +2,34 @@ schema:
   1:
     create_table_messages: >
       CREATE TABLE IF NOT EXISTS messages(
-        id INTEGER PRIMARY KEY autoincrement, 
-        channel TEXT NOT NULL, 
-        value TEXT NOT NULL, 
-        pid INTEGER, 
+        id INTEGER PRIMARY KEY autoincrement,
+        channel TEXT NOT NULL,
+        value TEXT NOT NULL,
+        pid INTEGER,
         created_at INTEGER NOT NULL ON CONFLICT REPLACE DEFAULT(unixepoch())
       );
     create_index_messages_by_date: >
-      CREATE INDEX IF NOT EXISTS messages_by_date ON messages(created_at);  
+      CREATE INDEX IF NOT EXISTS messages_by_date ON messages(created_at);
 
 stmts:
+  publish: >
+    INSERT INTO messages(channel, value, pid)
+    VALUES ($1, $2, $3);
 
-  publish: INSERT INTO messages(channel, value, pid) VALUES ($1, $2, $3)
+  last_id: >
+    SELECT max(id) FROM messages;
 
-  last_id: SELECT max(id) FROM messages
-  
-  fetch: SELECT id, channel, value, created_at FROM messages WHERE id > $1 and pid != $2
+  fetch: >
+    SELECT id, channel, value, created_at
+    FROM messages
+    WHERE id > $1
+      AND pid != $2;
 
-  prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
+  prune: >
+    DELETE FROM messages
+    WHERE created_at < (unixepoch() - $1);
 
-  check_prune: SELECT count(*) FROM messages WHERE created_at < (unixepoch() - $1)
+  check_prune: >
+    SELECT count(*)
+    FROM messages
+    WHERE created_at < (unixepoch() - $1);

data/lib/litestack/litecache.rb

@@ -1,33 +1,32 @@
 # frozen_stringe_literal: true
 
 # all components should require the support module
-require_relative 'litesupport'
-require_relative 'litemetric'
+require_relative "litesupport"
+require_relative "litemetric"
 
 ##
-#Litecache is a caching library for Ruby applications that is built on top of SQLite. It is designed to be simple to use, very fast, and feature-rich, providing developers with a reliable and efficient way to cache data.
+# Litecache is a caching library for Ruby applications that is built on top of SQLite. It is designed to be simple to use, very fast, and feature-rich, providing developers with a reliable and efficient way to cache data.
 #
-#One of the main features of Litecache is automatic key expiry, which allows developers to set an expiration time for each cached item. This ensures that cached data is automatically removed from the cache after a certain amount of time has passed, reducing the risk of stale data being served to users.
+# One of the main features of Litecache is automatic key expiry, which allows developers to set an expiration time for each cached item. This ensures that cached data is automatically removed from the cache after a certain amount of time has passed, reducing the risk of stale data being served to users.
 #
-#In addition, Litecache supports LRU (Least Recently Used) removal, which means that if the cache reaches its capacity limit, the least recently used items will be removed first to make room for new items. This ensures that the most frequently accessed data is always available in the cache.
+# In addition, Litecache supports LRU (Least Recently Used) removal, which means that if the cache reaches its capacity limit, the least recently used items will be removed first to make room for new items. This ensures that the most frequently accessed data is always available in the cache.
 #
-#Litecache also supports integer value increment/decrement, which allows developers to increment or decrement the value of a cached item in a thread-safe manner. This is useful for implementing counters or other types of numerical data that need to be updated frequently.
+# Litecache also supports integer value increment/decrement, which allows developers to increment or decrement the value of a cached item in a thread-safe manner. This is useful for implementing counters or other types of numerical data that need to be updated frequently.
 #
-#Overall, Litecache is a powerful and flexible caching library that provides automatic key expiry, LRU removal, and integer value increment/decrement capabilities. Its fast performance and simple API make it an excellent choice for Ruby applications that need a reliable and efficient way to cache data.
+# Overall, Litecache is a powerful and flexible caching library that provides automatic key expiry, LRU removal, and integer value increment/decrement capabilities. Its fast performance and simple API make it an excellent choice for Ruby applications that need a reliable and efficient way to cache data.
 
 class Litecache
-
   include Litesupport::Liteconnection
   include Litemetric::Measurable
 
   # the default options for the cache
-  # can be overriden by passing new options in a hash 
+  # can be overriden by passing new options in a hash
   # to Litecache.new
   #   path: "./cache.db"
   #   expiry: 60 * 60 * 24 * 30 -> one month default expiry if none is provided
-  #   size: 128 * 1024 * 1024 -> 128MB 
+  #   size: 128 * 1024 * 1024 -> 128MB
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
-  #   min_size: 32 * 1024 -> 32MB
+  #   min_size: 32 * 1024 -> 32KB
   #   return_full_record: false -> only return the payload
   #   sleep_interval: 1 -> 1 second of sleep between cleanup runs
 
@@ -35,15 +34,15 @@ class Litecache
     path: Litesupport.root.join("cache.sqlite3"),
     config_path: "./litecache.yml",
     sync: 0,
-    expiry: 60 * 60 * 24 * 30, # one month 
-    size: 128 * 1024 * 1024, #128MB 
-    mmap_size: 128 * 1024 * 1024, #128MB
-    min_size: 8  * 1024 * 1024, #16MB
-    return_full_record: false, #only return the payload
-    sleep_interval: 1, # 1 second
+    expiry: 60 * 60 * 24 * 30, # one month
+    size: 128 * 1024 * 1024, # 128MB
+    mmap_size: 128 * 1024 * 1024, # 128MB
+    min_size: 8 * 1024 * 1024, # 16MB
+    return_full_record: false, # only return the payload
+    sleep_interval: 30, # 30 seconds
     metrics: false
   }
-  
+
   # creates a new instance of Litecache
   # can optionally receive an options hash which will be merged
   # with the DEFAULT_OPTIONS (the new hash overrides any matching keys in the default one).
@@ -61,58 +60,72 @@ class Litecache
   #
   #   litecache.clear # nothing remains in the cache
   #   litecache.close # optional, you can safely kill the process
-  
+
   def initialize(options = {})
-    options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]        
-    @last_visited = {}
-    init(options)    
+    options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]
+    init(options)
+    @expires_in = @options[:expiry] || 60 * 60 * 24 * 30
     collect_metrics if @options[:metrics]
   end
-  
-  # add a key, value pair to the cache, with an optional expiry value (number of seconds)      
+
+  # add a key, value pair to the cache, with an optional expiry value (number of seconds)
   def set(key, value, expires_in = nil)
     key = key.to_s
-    expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
+    expires_in ||= @expires_in 
     @conn.acquire do |cache|
-      begin
-        cache.stmts[:setter].execute!(key, value, expires_in)
-        capture(:set, key)
-      rescue SQLite3::FullException
-        cache.stmts[:extra_pruner].execute!(0.2)
-        cache.execute("vacuum")
-        retry
-      end
+      cache.stmts[:setter].execute!(key, value, expires_in)
+      capture(:set, key)
+    rescue SQLite3::FullException
+      cache.stmts[:extra_pruner].execute!(0.2)
+      cache.execute("vacuum")
+      retry
     end
-    return true
+    true
   end
   
-  # add a key, value pair to the cache, but only if the key doesn't exist, with an optional expiry value (number of seconds)      
+  # set multiple keys and values in one shot set_multi({k1: v1, k2: v2, ... })
+  def set_multi(keys_and_values, expires_in = nil)
+    expires_in ||= @expires_in 
+    transaction do |conn|
+      keys_and_values.each_pair do |k, v|
+        begin
+          key = k.to_s
+          conn.stmts[:setter].execute!(key, v, expires_in)
+          capture(:set, key)
+        rescue SQLite3::FullException
+          conn.stmts[:extra_pruner].execute!(0.2)
+          conn.execute("vacuum")
+          retry
+        end
+      end
+    end
+    true    
+  end 
+
+  # add a key, value pair to the cache, but only if the key doesn't exist, with an optional expiry value (number of seconds)
   def set_unless_exists(key, value, expires_in = nil)
     key = key.to_s
-    expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
+    expires_in ||= @expires_in 
     changes = 0
     @conn.acquire do |cache|
-      begin
-        cache.transaction(:immediate) do
-          cache.stmts[:inserter].execute!(key, value, expires_in)
-          changes = cache.changes
-        end
-        capture(:set, key)
-      rescue SQLite3::FullException
-        cache.stmts[:extra_pruner].execute!(0.2)
-        cache.execute("vacuum")
-        retry
+      cache.transaction(:immediate) do
+        cache.stmts[:inserter].execute!(key, value, expires_in)
+        changes = cache.changes
       end
+      capture(:set, key)
+    rescue SQLite3::FullException
+      cache.stmts[:extra_pruner].execute!(0.2)
+      cache.execute("vacuum")
+      retry
     end
-    return changes > 0
+    changes > 0
   end
-  
+
   # get a value by its key
   # if the key doesn't exist or it is expired then null will be returned
   def get(key)
     key = key.to_s
-    if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
-      @last_visited[key] = true
+    if (record = @conn.acquire { |cache| cache.stmts[:getter].execute!(key)[0] })
       capture(:get, key, 1)
       return record[1]
     end
@@ -120,6 +133,24 @@ class Litecache
     nil
   end
   
+  # get multiple values by their keys, a hash with values corresponding to the keys
+  # is returned,
+  def get_multi(*keys)
+    results = {}
+    transaction(:deferred) do |conn|
+      keys.length.times do |i|
+        key = keys[i].to_s
+        if (record = conn.stmts[:getter].execute!(key)[0]) 
+          results[keys[i]] = record[1] # use the original key format
+          capture(:get, key, 1)
+        else
+          capture(:get, key, 0) 
+        end
+      end
+    end
+    results
+  end
+
   # delete a key, value pair from the cache
   def delete(key)
     changes = 0
@@ -127,59 +158,59 @@ class Litecache
       cache.stmts[:deleter].execute!(key)
       changes = cache.changes
     end
-    return changes > 0
+    changes > 0
   end
-  
+
   # increment an integer value by amount, optionally add an expiry value (in seconds)
   def increment(key, amount, expires_in = nil)
-    expires_in = @expires_in unless expires_in
-    @conn.acquire{|cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
+    expires_in ||= @expires_in 
+    @conn.acquire { |cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
   end
-  
+
   # decrement an integer value by amount, optionally add an expiry value (in seconds)
   def decrement(key, amount, expires_in = nil)
     increment(key, -amount, expires_in)
   end
-  
+
   # delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
-  def prune(limit=nil)
+  def prune(limit = nil)
     @conn.acquire do |cache|
-      if limit and limit.is_a? Integer
+      if limit&.is_a? Integer
         cache.stmts[:limited_pruner].execute!(limit)
-      elsif limit and limit.is_a? Float
+      elsif limit&.is_a? Float
         cache.stmts[:extra_pruner].execute!(limit)
       else
-        cache.stmts[:pruner].execute!      
+        cache.stmts[:pruner].execute!
       end
     end
   end
-  
+
   # return the number of key, value pairs in the cache
   def count
     run_stmt(:counter)[0][0]
   end
-  
+
   # return the actual size of the cache file
-  #def size
+  # def size
   #  run_stmt(:sizer)[0][0]
-  #end
-  
+  # end
+
   # delete all key, value pairs in the cache
   def clear
     run_sql("delete FROM data")
   end
-  
+
   # close the connection to the cache file
   def close
     @running = false
     super
   end
-  
+
   # return the maximum size of the cache
   def max_size
     run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0].to_f / (1024 * 1024)
   end
-  
+
   def snapshot
     {
       summary: {
@@ -193,55 +224,49 @@ class Litecache
     }
   end
 
-  
   # low level access to SQLite transactions, use with caution
-  def transaction(mode, acquire=true)
-    return cache.transaction(mode){yield} unless acquire
+  def transaction(mode=:immediate)
     @conn.acquire do |cache|
-      cache.transaction(mode) do
+      if cache.transaction_active?
         yield
+      else
+        cache.transaction(mode) do
+          yield cache
+        end
       end
     end
   end
 
-  private 
-  
+  private
+
   def setup
     super # create connection
     @bgthread = spawn_worker # create backgroud pruner thread
   end
-    
+
   def spawn_worker
     Litescheduler.spawn do
       while @running
         @conn.acquire do |cache|
-          begin
-            cache.transaction(:immediate) do
-              @last_visited.delete_if do |k| # there is a race condition here, but not a serious one
-                cache.stmts[:toucher].execute!(k) || true
-              end
-              cache.stmts[:pruner].execute!             
-            end
-          rescue SQLite3::BusyException
-            retry
-          rescue SQLite3::FullException
-            cache.stmts[:extra_pruner].execute!(0.2)
-          rescue Exception
-            # database is closed
-          end
+          cache.stmts[:pruner].execute!
+        rescue SQLite3::BusyException
+          retry
+        rescue SQLite3::FullException
+          cache.stmts[:extra_pruner].execute!(0.2)
+        rescue Exception => e # standard:disable Lint/RescueException
+          # database is closed
         end
         sleep @options[:sleep_interval]
       end
     end
   end
-  
+
   def create_connection
     super("#{__dir__}/litecache.sql.yml") do |conn|
       conn.cache_size = 2000
-      conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
+      conn.journal_size_limit = [(@options[:size] / 2).to_i, @options[:min_size]].min
       conn.max_page_count = (@options[:size] / conn.page_size).to_i
       conn.case_sensitive_like = true
     end
-  end  
-  
+  end
 end

data/lib/litestack/litecache.sql.yml

@@ -1,28 +1,89 @@
 schema:
-  1:    
-    create_table_data: >  
-      CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)      
+  1:
+    create_table_data: >
+      CREATE TABLE IF NOT EXISTS data(
+        id TEXT PRIMARY KEY,
+        value ANY,
+        expires_in INTEGER,
+        last_used INTEGER
+      ) STRICT;
     create_expiry_index: >
-      CREATE index if not exists expiry_index on data (expires_in)
+      CREATE INDEX IF NOT EXISTS expiry_index ON data (expires_in);
     create_last_used_index: >
-      CREATE index if not exists last_used_index on data (last_used)
+      CREATE INDEX IF NOT EXISTS last_used_index ON data (last_used);
 
 stmts:
-  pruner: DELETE FROM data WHERE expires_in <= $1
-  extra_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))
-  limited_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)
-  toucher: UPDATE data SET  last_used = unixepoch('now') WHERE id = $1
+  pruner: >
+    DELETE FROM data WHERE expires_in <= unixepoch('now');
+
+  extra_pruner: >
+    DELETE FROM data WHERE id IN (
+      SELECT id
+      FROM data
+      ORDER BY last_used ASC
+      LIMIT (
+        SELECT CAST((count(*) * $1) AS int)
+        FROM data
+      )
+    );
+
+  limited_pruner: >
+    DELETE FROM data
+    WHERE id IN (
+      SELECT id
+      FROM data
+      ORDER BY last_used ASC
+      LIMIT $1
+    );
+
+  toucher: >
+    UPDATE data
+    SET last_used = unixepoch('now')
+    WHERE id = $1;
+
   setter: >
-    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
-    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in
-  inserter: > 
-    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
-    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')
-  finder: SELECT id FROM data WHERE id = $1
-  getter: SELECT id, value, expires_in FROM data WHERE id = $1
-  deleter: delete FROM data WHERE id = $1 returning value
-  incrementer: > 
-    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
-    UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in
-  counter: SELECT count(*) FROM data
-  sizer: SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count
+    INSERT INTO data (id, value, expires_in, last_used)
+    VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now'))
+    ON CONFLICT(id) DO UPDATE
+    SET
+      value = EXCLUDED.value,
+      last_used = EXCLUDED.last_used,
+      expires_in = EXCLUDED.expires_in;
+  
+  inserter: >
+    INSERT INTO data (id, value, expires_in, last_used)
+    VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now'))
+    ON CONFLICT(id) DO UPDATE
+    SET
+      value = EXCLUDED.value,
+      last_used = EXCLUDED.last_used,
+      expires_in = EXCLUDED.expires_in
+    WHERE id = $1
+      AND expires_in <= unixepoch('now');
+
+  finder: >
+    SELECT id FROM data WHERE id = $1;
+  
+  getter: >
+    SELECT id, value, expires_in FROM data WHERE id = $1 AND expires_in >= unixepoch('now');
+  
+  deleter: >
+    delete FROM data WHERE id = $1 RETURNING value;
+  
+  incrementer: >
+    INSERT INTO data (id, value, expires_in, last_used)
+    VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now'))
+    ON CONFLICT(id) DO UPDATE
+    SET
+      value = CAST(value AS int) + CAST(EXCLUDED.value AS int),
+      last_used = EXCLUDED.last_used,
+      expires_in = EXCLUDED.expires_in;
+  
+      
+  
+  counter: >
+    SELECT count(*) FROM data;
+  
+  sizer: >
+    SELECT size.page_size * count.page_count
+    FROM pragma_page_size() AS size, pragma_page_count() AS count;

data/lib/litestack/litecache.yml

@@ -1,6 +1,6 @@
 data: 123
 
-production: 
+production:
   data: abc
 
 developments: