litestack 0.1.8 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d08724a0b9293f55ebed24ba6d738b103795b69563a210c4454322fc382e174e
-  data.tar.gz: 01401bfca727b4ef9452a9efafba846c886845dd43c1d8f5f5ce84cee22ab61f
+  metadata.gz: aded5660c8623a899124bf67fbc10a6ccaa8d7f3ddedee663390d7161bbeb100
+  data.tar.gz: 995f3256dc8beaa01f906202639c057ddf0a8c401cf1daef3ba7de310ca7ccae
 SHA512:
-  metadata.gz: 729320670e62261596eabbfd8f8d931117507317127e9fd9e5a796928e7031418455ca351dd2de815d4860edc2e688c0bbdb79560d3907cb3545deb75a1b4fae
-  data.tar.gz: d75cc4f23694c726a361b0bc474f561b8ffc4db074ef275839917d4c82461aca7a1c4a0b0e939cd749a7e7afbb58de86f3de254a6adadfc7f6d91ea9021218f3
+  metadata.gz: 59764a40453a8e4a809ce079fea5e99263bd3db3570f45f7b7b85975f4c9ef63f7737a5e1d92b5122bd1fd3f582bafa185d362017a904cb3d9f0af6ac1237801
+  data.tar.gz: 3b59c9b49d965b0eb6820fa9cdb12924f65902323fa53e0d83d55338a7c3d8ef50c0a2259fa0b47121b42b1d896fd59d47772d6b92546d6bd4e84d34b89671d6

data/CHANGELOG.md

@@ -1,6 +1,19 @@
 ## [Unreleased]
 
-## [0.1.8] - 2022-03-08
+## [0.2.1] - 2023-05-08
+
+- Fix a race condition in Litecable
+
+## [0.2.0] - 2023-05-08
+
+- Litecable, a SQLite driver for ActionCable
+- Litemetric for metrics collection support (experimental, disabled by default)
+- New schema for Litejob, old jobs are auto-migrated
+- Code refactoring, extraction of SQL statements to external files
+- Graceful shutdown support working properly
+- Fork resilience
+
+## [0.1.8] - 2023-03-08
 
 - More code cleanups, more test coverage
 - Retry support for jobs in Litejob
@@ -8,20 +21,20 @@
 - Initial graceful shutdown support for Litejob (incomplete)
 - More configuration options for Litejob
 
-## [0.1.7] - 2022-03-05
+## [0.1.7] - 2023-03-05
 
 - Code cleanup, removal of references to older name
 - Fix for the litedb rake tasks (thanks: netmute)
 - More fixes for the new concurrency model
 - Introduced a logger for the Litejobqueue (doesn't work with Polyphony, fix should come soon)
 
-## [0.1.6] - 2022-03-03
+## [0.1.6] - 2023-03-03
 
 - Revamped the locking model, more robust, minimal performance hit
 - Introduced a new resource pooling class
 - Litecache and Litejob now use the resource pool
 - Much less memory usage for Litecache and Litejob
 
-## [0.1.0] - 2022-02-26
+## [0.1.0] - 2023-02-26
 
 - Initial release

data/README.md

@@ -16,12 +16,14 @@ litestack provides integration with popular libraries, including:
 - ActiveRecord
 - ActiveSupport::Cache
 - ActiveJob
+- ActionCable
 
 With litestack you only need to add a single gem to your app which would replace a host of other gems and services, for example, a typical Rails app using litestack will no longer need the following services:
 
 - Database Server (e.g. PostgreSQL, MySQL)
 - Cache Server (e.g. Redis, Memcached)
 - Job Processor (e.g. Sidekiq, Goodjob)
+- Pubsub Server (e.g. Redis, PostgreSQL)
 
 To make it even more efficient, litestack will detect the presence of Fiber based IO frameworks like Async (e.g. when you use the Falcon web server) or Polyphony. It will then switch its background workers for caches and queues to fibers (using the semantics of the existing framework). This is done transparently and will generally lead to lower CPU and memory utilization.
 
@@ -50,6 +52,7 @@ litestack currently offers three main components
 - litedb
 - litecache
 - litejob
+- litecable
 
 > ![litedb](https://github.com/oldmoe/litestack/blob/master/assets/litedb_logo_teal.png?raw=true)
 
@@ -113,6 +116,8 @@ litecache spawns a background thread for cleanup purposes. In case it detects th
 
 > ![litejob](https://github.com/oldmoe/litestack/blob/master/assets/litejob_logo_teal.png?raw=true)
 
+More info about Litejob can be found in the [litejob guide](https://github.com/oldmoe/litestack/wiki/Litejob-guide)
+
 litejob is a fast and very efficient job queue processor for Ruby applications. It builds on top of SQLite as well, which provides transactional guarantees, persistence and exceptional performance. 
 
 #### Direct litejob usage
@@ -159,6 +164,28 @@ queues:
 
 The queues need to include a name and a priority (a number between 1 and 10) and can also optionally add the token "spawn", which means every job will run it its own concurrency context (thread or fiber)
 
+> ![litecable](https://github.com/oldmoe/litestack/blob/master/assets/litecable_logo_teal.png?raw=true)
+
+#### ActionCable
+
+This is a drop in replacement adapter for actioncable that replaces `async` and other production adapters (e.g. PostgreSQL, Redis). This adapter is currently only tested in local (inline) mode.
+
+Getting up and running with litecable requires configuring your cable.yaml file under the config/ directory
+
+cable.yaml
+```yaml
+development:
+  adapter: litecable
+
+test:
+  adapter: test
+
+staging:
+  adapter: litecable
+
+production:
+  adapter: litecable
+```
 
 ## Contributing
 

data/bench/bench_cache_raw.rb

@@ -8,7 +8,7 @@ require 'async/scheduler'
 Fiber.set_scheduler Async::Scheduler.new
 Fiber.scheduler.run
 
-require_relative '../lib/litestack'
+require_relative '../lib/litestack/litecache'
 #require 'litestack'
 
 cache = Litecache.new({path: '../db/cache.db'}) # default settings
@@ -16,10 +16,10 @@ redis = Redis.new # default settings
 
 values = []
 keys = []
-count = 5
+count = 1000
 count.times { keys << random_str(10) }
 
-[10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000].each do |size|
+[10, 100, 1000, 10000].each do |size|
   count.times do
     values << random_str(size) 
   end
@@ -35,8 +35,15 @@ count.times { keys << random_str(10) }
     cache.set(keys[i], values[i])
   end
 
+  #bench("file writes", count) do |i|
+  #  f = File.open("../files/#{keys[i]}.data", 'w+')
+  #  f.write(values[i])
+  #  f.close
+  #end
+  
+
   bench("Redis writes", count) do |i|
-    #redis.set(keys[i], values[i])
+    redis.set(keys[i], values[i])
   end
 
   puts "== Reads =="
@@ -44,8 +51,12 @@ count.times { keys << random_str(10) }
     cache.get(random_keys[i])
   end
 
+  #bench("file reads", count) do |i|
+  #  data = File.read("../files/#{keys[i]}.data")
+  #end
+
   bench("Redis reads", count) do |i|
-    #redis.get(random_keys[i])
+    redis.get(random_keys[i])
   end
   puts "=========================================================="
 

data/lib/action_cable/subscription_adapter/litecable.rb

@@ -0,0 +1,36 @@
+# frozen_stringe_literal: true
+
+require_relative '../../litestack/litecable'
+
+module ActionCable
+  module SubscriptionAdapter
+    class Litecable < ::Litecable# :nodoc:
+
+      attr_reader :logger, :server
+      
+      prepend ChannelPrefix
+
+      DEFAULT_OPTIONS = {
+        config_path: "./config/litecable.yml",
+        path: "./db/cable.db",
+        sync: 0, # no need to sync at all
+        mmap_size: 16 * 1024 * 1024, # 16MB of memory hold hot messages
+        expire_after: 10, # remove messages older than 10 seconds
+        listen_interval: 0.005, # check new messages every 5 milliseconds
+        metrics: false
+      }
+
+      def initialize(server, logger=nil)
+        @server = server
+        @logger = server.logger
+        super(DEFAULT_OPTIONS.dup)
+      end
+            
+      def shutdown
+        close
+      end
+    
+    end
+  end
+end
+

data/lib/active_job/queue_adapters/litejob_adapter.rb

@@ -7,21 +7,12 @@ require "active_job"
 
 module ActiveJob
   module QueueAdapters
-    # == Ultralite adapter for Active Job
+    # == Litestack adapter for Active Job
     #
     #
     #   Rails.application.config.active_job.queue_adapter = :litejob
     class LitejobAdapter
-      
-      DEFAULT_OPTIONS = {
-        config_path: "./config/litejob.yml",
-        path: "../db/queue.db",
-        queues: [["default", 1]],
-        logger: nil, # Rails performs its logging already
-        retries: 5, # It is recommended to stop retries at the Rails level
-        workers: 5
-      }      
-    
+          
       def initialize(options={})
         # we currently don't honour individual options per job class
         # possible in the future?
@@ -40,6 +31,15 @@ module ActiveJob
 
       class Job # :nodoc:
         
+        DEFAULT_OPTIONS = {
+          config_path: "./config/litejob.yml",
+          path: "../db/queue.db",
+          queues: [["default", 1]],
+          logger: nil, # Rails performs its logging already
+          retries: 5, # It is recommended to stop retries at the Rails level
+          workers: 5
+        }      
+
         include ::Litejob
   
         def perform(job_data)

data/lib/litestack/litecable.rb

@@ -0,0 +1,138 @@
+# frozen_stringe_literal: true
+
+# all components should require the support module
+require_relative 'litesupport'
+require_relative 'litemetric'
+
+require 'base64'
+require 'oj'
+
+class Litecable
+
+  include Litesupport::Liteconnection
+  include Litemetric::Measurable
+
+
+  DEFAULT_OPTIONS = {
+    config_path: "./litecable.yml",
+    path: "./cable.db",
+    sync: 0,
+    mmap_size: 16 * 1024 * 1024, # 16MB
+    expire_after: 5, # remove messages older than 5 seconds
+    listen_interval: 0.05, # check new messages every 50 milliseconds
+    metrics: false
+  }
+  
+  def initialize(options = {})  
+    @messages = []
+    init(options)
+  end
+  
+  # broadcast a message to a specific channel
+  def broadcast(channel, payload=nil)
+    # group meesages and only do broadcast every 10 ms
+    #run_stmt(:publish, channel.to_s, Oj.dump(payload), @pid)
+    # but broadcast locally normally
+    @mutex.synchronize{ @messages << [channel.to_s, Oj.dump(payload)] }
+    local_broadcast(channel, payload) 
+  end
+  
+  # subscribe to a channel, optionally providing a success callback proc
+  def subscribe(channel, subscriber, success_callback = nil)
+    @mutex.synchronize do
+      @subscribers[channel] = {} unless @subscribers[channel]
+      @subscribers[channel][subscriber] = true
+    end
+  end
+  
+  # unsubscribe from a channel
+  def unsubscribe(channel, subscriber)
+    @mutex.synchronize do
+      @subscribers[channel].delete(subscriber) rescue nil
+    end
+  end
+
+  private 
+    
+  def local_broadcast(channel, payload=nil)
+    return unless @subscribers[channel]
+    subscribers = []
+    @mutex.synchronize do
+      subscribers = @subscribers[channel].keys
+    end
+    subscribers.each do |subscriber|
+      subscriber.call(payload)      
+    end
+  end  
+      
+  def setup
+    super # create connection
+    @pid = Process.pid
+    @subscribers = {}
+    @mutex = Litesupport::Mutex.new
+    @running = true
+    @listener = create_listener
+    @pruner = create_pruner
+    @broadcaster = create_broadcaster
+    @last_fetched_id = nil
+  end
+
+  def create_broadcaster
+    Litesupport.spawn do
+      while @running do
+        @mutex.synchronize do
+          if @messages.length > 0
+            run_sql("BEGIN IMMEDIATE")
+            while msg = @messages.shift
+              run_stmt(:publish, msg[0], msg[1], @pid)
+            end
+            run_sql("END")
+          end
+        end
+        sleep 0.02
+      end      
+    end
+  end
+
+  def create_pruner
+    Litesupport.spawn do
+      while @running do
+        run_stmt(:prune, @options[:expire_after])
+        sleep @options[:expire_after]
+      end      
+    end
+  end
+
+  def create_listener
+    Litesupport.spawn do
+      while @running do
+        @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
+        @logger.info @last_fetched_id
+        run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
+          @logger.info "RECEIVED #{msg}"
+          @last_fetched_id = msg[0]
+          local_broadcast(msg[1], Oj.load(msg[2])) 
+        end
+        sleep @options[:listen_interval]
+      end      
+    end
+  end
+
+  def create_connection
+    conn = super
+    conn.wal_autocheckpoint = 10000 
+    sql = YAML.load_file("#{__dir__}/litecable.sql.yml")
+    version = conn.get_first_value("PRAGMA user_version")
+    sql["schema"].each_pair do |v, obj| 
+      if v > version
+        conn.transaction do 
+          obj.each{|k, s| conn.execute(s)}
+          conn.user_version = v
+        end
+      end
+    end 
+    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
+    conn
+  end
+  
+end

data/lib/litestack/litecable.sql.yml

@@ -0,0 +1,24 @@
+schema:
+  1:
+    create_table_messages: >
+      CREATE TABLE IF NOT EXISTS messages(
+        id INTEGER PRIMARY KEY autoincrement, 
+        channel TEXT NOT NULL, 
+        value TEXT NOT NULL, 
+        pid INTEGER, 
+        created_at INTEGER NOT NULL ON CONFLICT REPLACE DEFAULT(unixepoch())
+      );
+    create_index_messages_by_date: >
+      CREATE INDEX IF NOT EXISTS messages_by_date ON messages(created_at);  
+
+stmts:
+
+  publish: INSERT INTO messages(channel, value, pid) VALUES ($1, $2, $3)
+
+  last_id: SELECT max(id) FROM messages
+  
+  fetch: SELECT id, channel, value FROM messages WHERE id > $1 and pid != $2
+
+  prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
+
+  check_prune: SELECT count(*) FROM messages WHERE created_at < (unixepoch() - $1)

data/lib/litestack/litecache.rb

@@ -2,6 +2,7 @@
 
 # all components should require the support module
 require_relative 'litesupport'
+require_relative 'litemetric'
 
 ##
 #Litecache is a caching library for Ruby applications that is built on top of SQLite. It is designed to be simple to use, very fast, and feature-rich, providing developers with a reliable and efficient way to cache data.
@@ -16,6 +17,9 @@ require_relative 'litesupport'
 
 class Litecache
 
+  include Litesupport::Liteconnection
+  include Litemetric::Measurable
+
   # the default options for the cache
   # can be overriden by passing new options in a hash 
   # to Litecache.new
@@ -29,12 +33,15 @@ class Litecache
 
   DEFAULT_OPTIONS = {
     path: "./cache.db",
+    config_path: "./litecache.yml",
+    sync: 0,
     expiry: 60 * 60 * 24 * 30, # one month 
     size: 128 * 1024 * 1024, #128MB 
     mmap_size: 128 * 1024 * 1024, #128MB
-    min_size: 32 * 1024, #32MB
+    min_size: 8  * 1024 * 1024, #16MB
     return_full_record: false, #only return the payload
-    sleep_interval: 1 # 1 second
+    sleep_interval: 1, # 1 second
+    metrics: false
   }
   
   # creates a new instance of Litecache
@@ -56,36 +63,20 @@ class Litecache
   #   litecache.close # optional, you can safely kill the process
   
   def initialize(options = {})
-    @options = DEFAULT_OPTIONS.merge(options)
-    @options[:size] = @options[:min_size] if @options[:size] < @options[:min_size]
-    @sql = {
-      :pruner => "DELETE FROM data WHERE expires_in <= $1",
-      :extra_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))",
-      :limited_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)",
-      :toucher => "UPDATE data SET  last_used = unixepoch('now') WHERE id = $1",
-      :setter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in",
-      :inserter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')",
-      :finder => "SELECT id FROM data WHERE id = $1",
-      :getter => "SELECT id, value, expires_in FROM data WHERE id = $1",
-      :deleter => "delete FROM data WHERE id = $1 returning value",
-      :incrementer => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in",
-      :counter => "SELECT count(*) FROM data",
-      :sizer => "SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count" 
-    }
-    @cache = Litesupport::Pool.new(1){create_db}
-    @stats = {hit: 0, miss: 0}
+    options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]        
+    init(options)    
     @last_visited = {}
-    @running = true
-    @bgthread = spawn_worker 
+    collect_metrics if @options[:metrics]
   end
   
   # add a key, value pair to the cache, with an optional expiry value (number of seconds)      
   def set(key, value, expires_in = nil)
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       begin
         cache.stmts[:setter].execute!(key, value, expires_in)
+        capture(:write, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -100,12 +91,13 @@ class Litecache
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
     changes = 0
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       begin
-        transaction(:immediate) do
+        cache.transaction(:immediate) do
           cache.stmts[:inserter].execute!(key, value, expires_in)
-          changes = @cache.changes
+          changes = cache.changes
         end
+        capture(:write, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -119,19 +111,19 @@ class Litecache
   # if the key doesn't exist or it is expired then null will be returned
   def get(key)
     key = key.to_s
-    if record = @cache.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
+    if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
       @last_visited[key] = true
-      @stats[:hit] +=1
+      capture(:hit, key)
       return record[1]
     end
-    @stats[:miss] += 1
+    capture(:miss, key)
     nil
   end
   
   # delete a key, value pair from the cache
   def delete(key)
     changes = 0
-    @cache.aquire do |cache|
+    @conn.acquire do |cache|
       cache.stmts[:deleter].execute!(key)
       changes = cache.changes
     end
@@ -141,7 +133,7 @@ class Litecache
   # increment an integer value by amount, optionally add an expiry value (in seconds)
   def increment(key, amount, expires_in = nil)
     expires_in = @expires_in unless expires_in
-    @cache.acquire{|cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
+    @conn.acquire{|cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
   end
   
   # decrement an integer value by amount, optionally add an expiry value (in seconds)
@@ -151,7 +143,7 @@ class Litecache
   
   # delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
   def prune(limit=nil)
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       if limit and limit.is_a? Integer
         cache.stmts[:limited_pruner].execute!(limit)
       elsif limit and limit.is_a? Float
@@ -164,42 +156,34 @@ class Litecache
   
   # return the number of key, value pairs in the cache
   def count
-    @cache.acquire{|cache| cache.stmts[:counter].execute!.to_a[0][0] }
+    run_stmt(:counter)[0][0]
   end
   
   # return the actual size of the cache file
   def size
-    @cache.acquire{|cache| cache.stmts[:sizer].execute!.to_a[0][0] }
+    run_stmt(:sizer)[0][0]
   end
   
   # delete all key, value pairs in the cache
   def clear
-    @cache.acquire{|cache| cache.execute("delete FROM data") }
+    run_sql("delete FROM data")
   end
   
   # close the connection to the cache file
   def close
     @running = false
-    #Litesupport.synchronize do
-    @cache.acquire{|cache| cache.close }
-    #end
+    super
   end
   
   # return the maximum size of the cache
   def max_size
-    @cache.acquire{|cache| cache.get_first_value("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c") }
+    run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0]
   end
   
-  # hits and misses for get operations performed over this particular connection (not cache wide)
-  # 
-  #   litecache.stats # => {hit: 543, miss: 31} 
-  def stats
-    @stats
-  end
-
   # low level access to SQLite transactions, use with caution
-  def transaction(mode)
-    @cache.acquire do |cache|
+  def transaction(mode, acquire=true)
+    return cache.transaction(mode){yield} unless acquire
+    @conn.acquire do |cache|
       cache.transaction(mode) do
         yield
       end
@@ -208,10 +192,15 @@ class Litecache
 
   private 
   
+  def setup
+    super # create connection
+    @bgthread = spawn_worker # create backgroud pruner thread
+  end
+    
   def spawn_worker
     Litesupport.spawn do
       while @running
-        @cache.acquire do |cache|
+        @conn.acquire do |cache|
           begin
             cache.transaction(:immediate) do
               @last_visited.delete_if do |k| # there is a race condition here, but not a serious one
@@ -232,19 +221,24 @@ class Litecache
     end
   end
   
-  def create_db
-    db = Litesupport.create_db(@options[:path])
-    db.synchronous = 0
-    db.cache_size = 2000
-    db.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
-    db.mmap_size = @options[:mmap_size]
-    db.max_page_count = (@options[:size] / db.page_size).to_i
-    db.case_sensitive_like = true
-    db.execute("CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)")
-    db.execute("CREATE index if not exists expiry_index on data (expires_in)")
-    db.execute("CREATE index if not exists last_used_index on data (last_used)")
-    @sql.each_pair{|k, v| db.stmts[k] = db.prepare(v)}
-    db
+  def create_connection
+    conn = super 
+    conn.cache_size = 2000
+    conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
+    conn.max_page_count = (@options[:size] / conn.page_size).to_i
+    conn.case_sensitive_like = true
+    sql = YAML.load_file("#{__dir__}/litecache.sql.yml")
+    version = conn.get_first_value("PRAGMA user_version")
+    sql["schema"].each_pair do |v, obj| 
+      if v > version
+        conn.transaction do 
+          obj.each{|k, s| conn.execute(s)}
+          conn.user_version = v
+        end
+      end
+    end 
+    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
+    conn
   end
   
 end

data/lib/litestack/litecache.sql.yml

@@ -0,0 +1,28 @@
+schema:
+  1:    
+    create_table_data: >  
+      CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)      
+    create_expiry_index: >
+      CREATE index if not exists expiry_index on data (expires_in)
+    create_last_used_index: >
+      CREATE index if not exists last_used_index on data (last_used)
+
+stmts:
+  pruner: DELETE FROM data WHERE expires_in <= $1
+  extra_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))
+  limited_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)
+  toucher: UPDATE data SET  last_used = unixepoch('now') WHERE id = $1
+  setter: >
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in
+  inserter: > 
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')
+  finder: SELECT id FROM data WHERE id = $1
+  getter: SELECT id, value, expires_in FROM data WHERE id = $1
+  deleter: delete FROM data WHERE id = $1 returning value
+  incrementer: > 
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in
+  counter: SELECT count(*) FROM data
+  sizer: SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count