litestack 0.4.2 → 0.4.4

data/gemfiles/rails71.gemfile.lock

@@ -0,0 +1,264 @@
+PATH
+  remote: ..
+  specs:
+    litestack (0.4.4)
+      erubi
+      oj
+      rack
+      rackup
+      sqlite3
+      tilt
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+      zeitwerk (~> 2.6)
+    actionmailbox (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      activejob (= 7.1.3.2)
+      activerecord (= 7.1.3.2)
+      activestorage (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      mail (>= 2.7.1)
+      net-imap
+      net-pop
+      net-smtp
+    actionmailer (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      actionview (= 7.1.3.2)
+      activejob (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      mail (~> 2.5, >= 2.5.4)
+      net-imap
+      net-pop
+      net-smtp
+      rails-dom-testing (~> 2.2)
+    actionpack (7.1.3.2)
+      actionview (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    actiontext (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      activerecord (= 7.1.3.2)
+      activestorage (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.1.3.2)
+      activesupport (= 7.1.3.2)
+      builder (~> 3.1)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activejob (7.1.3.2)
+      activesupport (= 7.1.3.2)
+      globalid (>= 0.3.6)
+    activemodel (7.1.3.2)
+      activesupport (= 7.1.3.2)
+    activerecord (7.1.3.2)
+      activemodel (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      timeout (>= 0.4.0)
+    activestorage (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      activejob (= 7.1.3.2)
+      activerecord (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      marcel (~> 1.0)
+    activesupport (7.1.3.2)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.6)
+    builder (3.2.4)
+    concurrent-ruby (1.2.3)
+    connection_pool (2.4.1)
+    crass (1.0.6)
+    date (3.3.4)
+    docile (1.4.0)
+    drb (2.2.1)
+    erubi (1.12.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    hanami-router (0.6.2)
+      hanami-utils (~> 0.7)
+      http_router (~> 0.11)
+    hanami-utils (0.9.2)
+    http_router (0.11.2)
+      rack (>= 1.0.0)
+      url_mount (~> 0.2.1)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    io-console (0.7.2)
+    irb (1.11.2)
+      rdoc
+      reline (>= 0.4.2)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    lint_roller (1.1.0)
+    loofah (2.22.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.4)
+    mini_mime (1.1.5)
+    minitest (5.22.2)
+    mutex_m (0.2.0)
+    net-imap (0.4.10)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.4.0.1)
+      net-protocol
+    nio4r (2.7.0)
+    nokogiri (1.16.2-arm64-darwin)
+      racc (~> 1.4)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
+    parallel (1.24.0)
+    parser (3.3.0.5)
+      ast (~> 2.4.1)
+      racc
+    psych (5.1.2)
+      stringio
+    racc (1.7.3)
+    rack (3.0.9.1)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails (7.1.3.2)
+      actioncable (= 7.1.3.2)
+      actionmailbox (= 7.1.3.2)
+      actionmailer (= 7.1.3.2)
+      actionpack (= 7.1.3.2)
+      actiontext (= 7.1.3.2)
+      actionview (= 7.1.3.2)
+      activejob (= 7.1.3.2)
+      activemodel (= 7.1.3.2)
+      activerecord (= 7.1.3.2)
+      activestorage (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      bundler (>= 1.15.0)
+      railties (= 7.1.3.2)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.1.3.2)
+      actionpack (= 7.1.3.2)
+      activesupport (= 7.1.3.2)
+      irb
+      rackup (>= 1.0.0)
+      rake (>= 12.2)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rainbow (3.1.1)
+    rake (13.1.0)
+    rdoc (6.6.2)
+      psych (>= 4.0.0)
+    regexp_parser (2.9.0)
+    reline (0.4.3)
+      io-console (~> 0.5)
+    rexml (3.2.6)
+    rubocop (1.61.0)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    ruby-progressbar (1.13.0)
+    sequel (5.78.0)
+      bigdecimal
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (1.7.2-arm64-darwin)
+    standard (1.34.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.60)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.1)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.2)
+    stringio (3.1.0)
+    thor (1.3.1)
+    tilt (2.3.0)
+    timeout (0.4.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
+    url_mount (0.2.1)
+      rack
+    webrick (1.8.1)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.6.13)
+
+PLATFORMS
+  arm64-darwin
+
+DEPENDENCIES
+  litestack!
+  minitest
+  rack (~> 3.0)
+  rails (~> 7.1)
+  railties
+  rake
+  sequel
+  simplecov
+  standard
+
+BUNDLED WITH
+   2.5.3

data/lib/active_job/queue_adapters/litejob_adapter.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require_relative "../../litestack/litejob"
+require "active_support"
 require "active_support/core_ext/enumerable"
+require "active_support/core_ext/numeric/time"
 require "active_support/core_ext/array/access"
 require "active_job"
 
@@ -18,20 +20,26 @@ module ActiveJob
         # Job.options = DEFAULT_OPTIONS.merge(options)
       end
 
+      def enqueue_after_transaction_commit?
+        Job.options[:enqueue_after_transaction_commit]
+      end
+
       def enqueue(job) # :nodoc:
         Job.queue = job.queue_name
         Job.perform_async(job.serialize)
       end
 
-      def enqueue_at(job, timestamp) # :nodoc:
+      def enqueue_at(job, time) # :nodoc:
+        time  = time.from_now if time.respond_to?(:from_now) #is_a?(ActiveSupport::Duration)
         Job.queue = job.queue_name
-        Job.perform_at(timestamp, job.serialize)
+        Job.perform_at(time, job.serialize)
       end
 
       class Job # :nodoc:
         DEFAULT_OPTIONS = {
           config_path: "./config/litejob.yml",
-          logger: nil # Rails performs its logging already
+          logger: nil, # Rails performs its logging already
+          enqueue_after_transaction_commit: true
         }
 
         include ::Litejob

data/lib/active_record/connection_adapters/litedb_adapter.rb

@@ -1,4 +1,5 @@
 require_relative "../../litestack/litedb"
+
 require "active_record"
 require "active_record/connection_adapters/sqlite3_adapter"
 require "active_record/tasks/sqlite_database_tasks"
@@ -90,3 +91,10 @@ module ActiveRecord
     end
   end
 end
+
+ActiveRecord::ConnectionAdapters.register(
+  "litedb", "ActiveRecord::ConnectionAdapters::LitedbAdapter", 
+  "active_record/connection_adapters/litedb_adapter"
+) if ActiveRecord::ConnectionAdapters.respond_to?(:register)
+
+

data/lib/active_support/cache/litecache.rb

@@ -1,14 +1,16 @@
-require "delegate"
-require "active_support/core_ext/enumerable"
-require "active_support/core_ext/array/extract_options"
+require "active_support"
 require "active_support/core_ext/numeric/time"
 require "active_support/cache"
+
 require_relative "../../litestack/litecache"
 
 module ActiveSupport
   module Cache
+
+    self.format_version = 7.0
+
     class Litecache < Store
-      prepend Strategy::LocalCache
+      # prepend Strategy::LocalCache
 
       def self.supports_cache_versioning?
         true
@@ -25,17 +27,22 @@ module ActiveSupport
         options = merged_options(options)
         # todo: fix me
         # this is currently a hack to avoid dealing with Rails cache encoding and decoding
+        # and it can result in a race condition as it stands
         # @cache.transaction(:immediate) do
+        # currently transactions are not compatible with acquiring connections
+        # this needs fixing by storing the connection to the context once acquired
         if (value = read(key, options))
           value = value.to_i + amount
           write(key, value, options)
+        else
+          write(key, amount, options)
         end
         # end
       end
 
       def decrement(key, amount = 1, options = nil)
         options = merged_options(options)
-        increment(key, -1 * amount, options[:expires_in])
+        increment(key, -1 * amount, options)
       end
 
       def prune(limit = nil, time = nil)
@@ -46,7 +53,7 @@ module ActiveSupport
         @cache.prune(limit)
       end
 
-      def clear
+      def clear(options = nil)
         @cache.clear
       end
 
@@ -68,18 +75,44 @@ module ActiveSupport
 
       private
 
+      def serialize_entries(entry, **options)
+        Marshal.dump(entry)
+      end
+
+      def deserialize_entries(entry)
+        Marshal.load(entry.to_s)
+      end
+
       # Read an entry from the cache.
       def read_entry(key, **options)
         deserialize_entry(@cache.get(key))
       end
 
+      def read_multi_entries(names, **options)
+        results = {}
+        return results if names == []
+        rs = @cache.get_multi(*names.flatten)
+        rs.each_pair { |k, v| results[k] = deserialize_entry(v).value }
+        results
+      end
+
       # Write an entry to the cache.
       def write_entry(key, entry, **options)
         write_serialized_entry(key, serialize_entry(entry, **options), **options)
       end
 
+      def write_multi_entries(entries, **options)
+        return if entries.empty?
+        entries.each_pair { |k, v| entries[k] = serialize_entry(v, **options) }
+        expires_in = options[:expires_in].to_i if options[:expires_in]
+        if options[:race_condition_ttl] && expires_in > 0 && !options[:raw]
+          expires_in += 5.minutes
+        end
+        @cache.set_multi(entries, expires_in)
+      end
+
       def write_serialized_entry(key, payload, **options)
-        expires_in = options[:expires_in].to_i
+        expires_in = options[:expires_in].to_i if options[:expires_in]
         if options[:race_condition_ttl] && expires_in > 0 && !options[:raw]
           expires_in += 5.minutes
         end

data/lib/generators/litestack/install/install_generator.rb

@@ -1,10 +1,10 @@
 class Litestack::InstallGenerator < Rails::Generators::Base
   source_root File.expand_path("templates", __dir__)
 
-  # Force copy configuratioon files so Rails installs don't ask questions
+  # Force copy configuration files so Rails installs don't ask questions
   # that less experienced people might not understand. The more Sr folks.
   # will know to check git to look at what changed.
-  def modify_database_adapater
+  def modify_database_adapter
     copy_file "database.yml", "config/database.yml", force: true
   end
 

data/lib/generators/litestack/install/templates/cable.yml

@@ -4,8 +4,5 @@ development:
 test:
   adapter: test
 
-staging:
-  adapter: litecable
-
 production:
   adapter: litecable

data/lib/litestack/liteboard/liteboard.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "hanami/router"
+require "rack"
 require "tilt"
 require "erubi"
 
@@ -11,37 +11,32 @@ class Liteboard
   @@resolutions = {"minute" => [300, 12], "hour" => [3600, 24], "day" => [3600 * 24, 7], "week" => [3600 * 24 * 7, 53], "year" => [3600 * 24 * 365, 100]}
   @@res_mapping = {"hour" => "minute", "day" => "hour", "week" => "day", "year" => "week"}
   @@templates = {}
-  @@app = Hanami::Router.new do
-    get "/", to: ->(env) do
+  @@app = proc do |env|
+    case path = env["PATH_INFO"]
+    when "/"
       Liteboard.new(env).call(:index)
-    end
-
-    get "/topics/Litejob", to: ->(env) do
+    when "/topics/Litejob"
       Liteboard.new(env).call(:litejob)
-    end
-
-    get "/topics/Litecache", to: ->(env) do
+    when "/topics/Litecache"
       Liteboard.new(env).call(:litecache)
-    end
-
-    get "/topics/Litedb", to: ->(env) do
+    when "/topics/Litedb"
       Liteboard.new(env).call(:litedb)
-    end
-
-    get "/topics/Litecable", to: ->(env) do
+    when "/topics/Litecable"
       Liteboard.new(env).call(:litecable)
     end
+
   end
 
   def initialize(env)
     @env = env
-    @params = @env["router.params"]
+    @req = Rack::Request.new(@env)
+    @params = @req.params 
     @running = true
     @lm = Litemetric.instance
   end
 
   def params(key)
-    URI.decode_uri_component(@params[key].to_s)
+    URI.decode_uri_component(@params[key.to_s].to_s)
   end
 
   def call(method)
@@ -227,9 +222,9 @@ class Liteboard
     @topic = "Litejob"
     @events = @lm.events_summaries(@topic, @resolution, @order, @dir, @search, @step * @count)
     @events.each do |event|
-      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event["name"])
+      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event[:name])
       event["counts"] = data_points.collect { |r| [r["rtime"], r["rcount"] || 0] }
-      event["values"] = data_points.collect { |r| [r["rtime"], r["rtotal"] || 0] }
+      event["values"] = data_points.collect { |r| [r["rtime"], (r["rtotal"] || 0.0)] }
     end
     @snapshot = read_snapshot(@topic)
     @size = begin
@@ -391,6 +386,7 @@ class Liteboard
   end
 
   def format(float)
+    float = float.round(3)
     string = float.to_s
     whole, decimal = string.split(".")
     whole = whole.chars.reverse.each_slice(3).map(&:join).join(",").reverse

data/lib/litestack/liteboard/views/litecable.erb

@@ -61,7 +61,7 @@
         </div>
         <div class="card-body">
           <span class="hidden inlinestackedcolumn">
-            <%=[["Time", "Recieved Count", "Delivered Count"]] + @messages_over_time.to_a%>
+            <%=[["Time", "Received Count", "Delivered Count"]] + @messages_over_time.to_a%>
           </span>
         </div>
       </div>

data/lib/litestack/litecable.rb

@@ -113,7 +113,7 @@ class Litecable
   def create_listener
     Litescheduler.spawn do
       while @running
-        @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
+        @last_fetched_id ||= run_stmt(:last_id)[0][0] || 0
         run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
           @logger.info "RECEIVED #{msg}"
           @last_fetched_id = msg[0]

data/lib/litestack/litecache.rb

@@ -20,13 +20,13 @@ class Litecache
   include Litemetric::Measurable
 
   # the default options for the cache
-  # can be overriden by passing new options in a hash
+  # can be overridden by passing new options in a hash
   # to Litecache.new
   #   path: "./cache.db"
   #   expiry: 60 * 60 * 24 * 30 -> one month default expiry if none is provided
   #   size: 128 * 1024 * 1024 -> 128MB
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
-  #   min_size: 32 * 1024 -> 32MB
+  #   min_size: 32 * 1024 -> 32KB
   #   return_full_record: false -> only return the payload
   #   sleep_interval: 1 -> 1 second of sleep between cleanup runs
 
@@ -39,7 +39,7 @@ class Litecache
     mmap_size: 128 * 1024 * 1024, # 128MB
     min_size: 8 * 1024 * 1024, # 16MB
     return_full_record: false, # only return the payload
-    sleep_interval: 1, # 1 second
+    sleep_interval: 30, # 30 seconds
     metrics: false
   }
 
@@ -63,15 +63,15 @@ class Litecache
 
   def initialize(options = {})
     options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]
-    @last_visited = {}
     init(options)
+    @expires_in = @options[:expiry] || 60 * 60 * 24 * 30
     collect_metrics if @options[:metrics]
   end
 
   # add a key, value pair to the cache, with an optional expiry value (number of seconds)
   def set(key, value, expires_in = nil)
     key = key.to_s
-    expires_in = @options[:expires_in] if expires_in.nil? || expires_in.zero?
+    expires_in ||= @expires_in
     @conn.acquire do |cache|
       cache.stmts[:setter].execute!(key, value, expires_in)
       capture(:set, key)
@@ -83,10 +83,27 @@ class Litecache
     true
   end
 
+  # set multiple keys and values in one shot set_multi({k1: v1, k2: v2, ... })
+  def set_multi(keys_and_values, expires_in = nil)
+    expires_in ||= @expires_in
+    transaction do |conn|
+      keys_and_values.each_pair do |k, v|
+        key = k.to_s
+        conn.stmts[:setter].execute!(key, v, expires_in)
+        capture(:set, key)
+      rescue SQLite3::FullException
+        conn.stmts[:extra_pruner].execute!(0.2)
+        conn.execute("vacuum")
+        retry
+      end
+    end
+    true
+  end
+
   # add a key, value pair to the cache, but only if the key doesn't exist, with an optional expiry value (number of seconds)
   def set_unless_exists(key, value, expires_in = nil)
     key = key.to_s
-    expires_in = @options[:expires_in] if expires_in.nil? || expires_in.zero?
+    expires_in ||= @expires_in
     changes = 0
     @conn.acquire do |cache|
       cache.transaction(:immediate) do
@@ -107,7 +124,6 @@ class Litecache
   def get(key)
     key = key.to_s
     if (record = @conn.acquire { |cache| cache.stmts[:getter].execute!(key)[0] })
-      @last_visited[key] = true
       capture(:get, key, 1)
       return record[1]
     end
@@ -115,6 +131,24 @@ class Litecache
     nil
   end
 
+  # get multiple values by their keys, a hash with values corresponding to the keys
+  # is returned,
+  def get_multi(*keys)
+    results = {}
+    transaction(:deferred) do |conn|
+      keys.length.times do |i|
+        key = keys[i].to_s
+        if (record = conn.stmts[:getter].execute!(key)[0])
+          results[keys[i]] = record[1] # use the original key format
+          capture(:get, key, 1)
+        else
+          capture(:get, key, 0)
+        end
+      end
+    end
+    results
+  end
+
   # delete a key, value pair from the cache
   def delete(key)
     changes = 0
@@ -126,13 +160,13 @@ class Litecache
   end
 
   # increment an integer value by amount, optionally add an expiry value (in seconds)
-  def increment(key, amount, expires_in = nil)
+  def increment(key, amount = 1, expires_in = nil)
     expires_in ||= @expires_in
     @conn.acquire { |cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
   end
 
   # decrement an integer value by amount, optionally add an expiry value (in seconds)
-  def decrement(key, amount, expires_in = nil)
+  def decrement(key, amount = 1, expires_in = nil)
     increment(key, -amount, expires_in)
   end
 
@@ -189,11 +223,14 @@ class Litecache
   end
 
   # low level access to SQLite transactions, use with caution
-  def transaction(mode, acquire = true)
-    return cache.transaction(mode) { yield } unless acquire
+  def transaction(mode = :immediate)
     @conn.acquire do |cache|
-      cache.transaction(mode) do
+      if cache.transaction_active?
         yield
+      else
+        cache.transaction(mode) do
+          yield cache
+        end
       end
     end
   end
@@ -202,19 +239,14 @@ class Litecache
 
   def setup
     super # create connection
-    @bgthread = spawn_worker # create backgroud pruner thread
+    @bgthread = spawn_worker # create background pruner thread
   end
 
   def spawn_worker
     Litescheduler.spawn do
       while @running
         @conn.acquire do |cache|
-          cache.transaction(:immediate) do
-            @last_visited.delete_if do |k| # there is a race condition here, but not a serious one
-              cache.stmts[:toucher].execute!(k) || true
-            end
-            cache.stmts[:pruner].execute!
-          end
+          cache.stmts[:pruner].execute!
         rescue SQLite3::BusyException
           retry
         rescue SQLite3::FullException