honker 0.1.1 → 0.3.0

data/lib/honker.rb

@@ -18,6 +18,8 @@
 # is one SQL call via the `sqlite3` gem. No extra process, no Redis.
 
 require "json"
+require "fiddle"
+require "rbconfig"
 require "sqlite3"
 
 require_relative "honker/version"
@@ -25,12 +27,137 @@ require_relative "honker/transaction"
 require_relative "honker/stream"
 require_relative "honker/scheduler"
 require_relative "honker/lock"
+require_relative "honker/railtie" if defined?(::Rails::Railtie)
 
 module Honker
+  # Honker's error class, raised by ExtensionResolver and CoreWatcher.
+  class Error < StandardError; end
+
+  # Resolves the path to the Honker SQLite loadable extension. Platform
+  # gems ship it bundled in lib/honker/; an explicit path and the
+  # HONKER_EXTENSION_PATH override take precedence.
+  class ExtensionResolver
+    def initialize(env: ENV.fetch("HONKER_EXTENSION_PATH", nil), bundled: nil)
+      @env = env
+      @bundled = bundled || File.expand_path("honker/#{extension_filename}", __dir__)
+    end
+
+    # Returns the extension path: an explicit `extension_path`, else
+    # HONKER_EXTENSION_PATH, else the bundled extension. Raises
+    # Honker::Error when HONKER_EXTENSION_PATH or the bundled extension
+    # is missing.
+    def resolve(extension_path = nil)
+      return extension_path unless extension_path.nil?
+      return env_extension unless env.nil? || env.empty?
+
+      path = bundled
+      return path if File.file?(path)
+
+      raise Error, "Honker SQLite extension not found at #{path}; " \
+                   "set HONKER_EXTENSION_PATH or pass extension_path:"
+    end
+
+    private
+
+    attr_reader :env, :bundled
+
+    def env_extension
+      return env if File.file?(env)
+
+      raise Error, "HONKER_EXTENSION_PATH does not exist: #{env}"
+    end
+
+    def extension_filename
+      case RbConfig::CONFIG.fetch("host_os")
+      when /mswin|mingw|cygwin/ then "honker_ext.dll"
+      when /darwin/ then "libhonker_ext.dylib"
+      else "libhonker_ext.so"
+      end
+    end
+  end
+
+  # Resolve the bundled (or overridden) extension path without naming
+  # ExtensionResolver — useful for `database.yml` ERB and tooling.
+  def self.extension_path(override = nil)
+    ExtensionResolver.new.resolve(override)
+  end
+
+  # Load the Honker extension onto a raw SQLite3::Database. Encapsulates
+  # the enable_load_extension(true)/load/enable_load_extension(false)
+  # sequence so the toggle-off can't be forgotten.
+  def self.load_extension(sqlite_conn, extension_path: nil)
+    resolved = ExtensionResolver.new.resolve(extension_path)
+    sqlite_conn.enable_load_extension(true)
+    sqlite_conn.load_extension(resolved)
+  ensure
+    sqlite_conn.enable_load_extension(false)
+  end
+
+  # Run honker_bootstrap() on the connection. Idempotent. Separated from
+  # load_extension so production users can opt to bootstrap from a
+  # migration instead of every connect.
+  def self.bootstrap(sqlite_conn)
+    sqlite_conn.execute("SELECT honker_bootstrap()")
+  end
+
+  # Convenience: load the extension then bootstrap. The one-liner most
+  # ORM integrations reach for.
+  def self.setup(sqlite_conn, extension_path: nil, bootstrap: true)
+    load_extension(sqlite_conn, extension_path: extension_path)
+    self.bootstrap(sqlite_conn) if bootstrap
+  end
+
+  # Returns a Proc suitable for Sequel/Rom/Hanami `after_connect:`.
+  def self.sequel_after_connect(extension_path: nil, bootstrap: true)
+    proc { |conn| setup(conn, extension_path: extension_path, bootstrap: bootstrap) }
+  end
+
+  class CoreWatcher
+    def initialize(db_path, extension_path, backend)
+      @lib = Fiddle.dlopen(extension_path)
+      @open = Fiddle::Function.new(
+        @lib["honker_watcher_open"],
+        [Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T],
+        Fiddle::TYPE_VOIDP,
+      )
+      @wait = Fiddle::Function.new(
+        @lib["honker_watcher_wait"],
+        [Fiddle::TYPE_VOIDP, Fiddle::TYPE_LONG_LONG],
+        Fiddle::TYPE_INT,
+      )
+      @close = Fiddle::Function.new(
+        @lib["honker_watcher_close"],
+        [Fiddle::TYPE_VOIDP],
+        Fiddle::TYPE_VOID,
+      )
+      err = "\0" * 1024
+      @handle = @open.call(db_path.to_s, backend.to_s, err, err.bytesize)
+      return unless @handle.to_i.zero?
+
+      raise ArgumentError, err.delete_suffix("\0").split("\0", 2).first
+    end
+
+    def wait(timeout_s)
+      code = @wait.call(@handle, (timeout_s * 1000).ceil)
+      return true if code == 1
+      return false if code == 0
+
+      raise Error, "honker update watcher closed or died"
+    end
+
+    def close
+      return if @handle.nil? || @handle.to_i.zero?
+
+      @close.call(@handle)
+      @handle = nil
+    end
+  end
+
   DEFAULT_PRAGMAS = <<~SQL
     PRAGMA journal_mode = WAL;
     PRAGMA synchronous = NORMAL;
     PRAGMA busy_timeout = 5000;
+    PRAGMA mmap_size = 0;
     PRAGMA foreign_keys = ON;
     PRAGMA cache_size = -32000;
     PRAGMA temp_store = MEMORY;
@@ -43,17 +170,27 @@ module Honker
   class Database
     attr_reader :db
 
-    def initialize(path, extension_path:)
+    def initialize(path, extension_path: nil, watcher_backend: nil,
+                   extension_resolver: ExtensionResolver.new)
+      unless watcher_backend.nil? || watcher_backend.is_a?(String)
+        raise ArgumentError, "unknown watcher backend"
+      end
+
+      resolved_extension = extension_resolver.resolve(extension_path)
       @db = SQLite3::Database.new(path)
       @local_update_seq = 0
+      @db.busy_timeout = 5000
+      @db.execute("PRAGMA mmap_size = 0")
       @db.enable_load_extension(true)
-      @db.load_extension(extension_path)
+      @db.load_extension(resolved_extension)
       @db.enable_load_extension(false)
       @db.execute_batch(DEFAULT_PRAGMAS)
       @db.execute("SELECT honker_bootstrap()")
+      @watcher = CoreWatcher.new(path, resolved_extension, watcher_backend)
     end
 
     def close
+      @watcher&.close
       @db&.close
     end
 
@@ -65,6 +202,10 @@ module Honker
       @local_update_seq
     end
 
+    def wait_for_update(timeout_s)
+      @watcher.wait(timeout_s)
+    end
+
     # Returns a Queue handle for a named queue.
     #
     #   visibility_timeout_s: 300   # claim expiry before reclaim
@@ -78,6 +219,18 @@ module Honker
       )
     end
 
+    # Transactional side-effect delivery built on a reserved queue.
+    def outbox(name, delivery, visibility_timeout_s: 60, max_attempts: 5, base_backoff_s: 5)
+      Outbox.new(
+        self,
+        name,
+        delivery,
+        visibility_timeout_s: visibility_timeout_s,
+        max_attempts: max_attempts,
+        base_backoff_s: base_backoff_s,
+      )
+    end
+
     # Returns a Stream handle for an append-only ordered log.
     def stream(name)
       Stream.new(self, name)
@@ -281,6 +434,87 @@ module Honker
         [job_id, worker_id, extend_s],
       )[0] == 1
     end
+
+    # Delete a pending or processing job by id. Returns true iff a row
+    # was removed. Idempotent on missing.
+    #
+    # IMPORTANT: cancel does NOT interrupt a worker currently running
+    # the handler. It invalidates the worker's claim — its next
+    # ack/heartbeat returns false. If you need the handler to actually
+    # halt, build that signal in your app.
+    def cancel(job_id)
+      n = @db.db.get_first_row("SELECT honker_cancel(?)", [job_id])[0]
+      @db.mark_updated if n.positive?
+      n.positive?
+    end
+
+    # Read a single job row by id. Returns a Hash with the row fields,
+    # or nil if the job has been ack'd, dead'd, or never existed.
+    def get_job(job_id)
+      raw = @db.db.get_first_row("SELECT honker_get_job(?)", [job_id])[0]
+      return nil if raw.nil? || raw.empty?
+
+      JSON.parse(raw)
+    end
+  end
+
+  class Outbox
+    attr_reader :name, :queue, :max_attempts, :base_backoff_s
+
+    def initialize(db, name, delivery, visibility_timeout_s:, max_attempts:, base_backoff_s:)
+      raise ArgumentError, "delivery must respond to #call" unless delivery.respond_to?(:call)
+
+      @name = name
+      @delivery = delivery
+      @max_attempts = max_attempts
+      @base_backoff_s = base_backoff_s
+      @queue = db.queue(
+        "_outbox:#{name}",
+        visibility_timeout_s: visibility_timeout_s,
+        max_attempts: max_attempts,
+      )
+    end
+
+    def enqueue(payload, tx: nil, delay: nil, run_at: nil, priority: 0, expires: nil)
+      if tx
+        @queue.enqueue_tx(tx, payload, delay: delay, run_at: run_at, priority: priority, expires: expires)
+      else
+        @queue.enqueue(payload, delay: delay, run_at: run_at, priority: priority, expires: expires)
+      end
+    end
+
+    def run_once(worker_id)
+      job = @queue.claim_one(worker_id)
+      return false unless job
+
+      begin
+        if @delivery.arity == 1
+          @delivery.call(job.payload)
+        else
+          @delivery.call(job.payload, job)
+        end
+        raise "outbox ack failed for job #{job.id}" unless job.ack
+      rescue StandardError => e
+        delay_s = retry_delay(job.attempts)
+        raise "outbox retry failed for job #{job.id}" unless job.retry(delay_s: delay_s, error: "#{e}\n#{e.backtrace&.join("\n")}")
+      end
+      true
+    end
+
+    def run_worker(worker_id, stop: nil, idle_sleep_s: 0.1)
+      until stop&.call
+        processed = run_once(worker_id)
+        sleep(idle_sleep_s) unless processed
+      end
+    end
+
+    private
+
+    def retry_delay(attempts)
+      return 0 if @base_backoff_s <= 0
+
+      (@base_backoff_s * (2**[attempts - 1, 0].max)).ceil
+    end
   end
 
   # A claimed unit of work. `payload` is the decoded JSON value (Hash,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: honker
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Russell Romney
@@ -9,44 +9,89 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 2.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 2.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 description: |-
   Ruby binding for Honker — a SQLite-native task runtime. Queues,
   streams, pub/sub, time-trigger scheduler, results, locks, rate limits, all
   in one .db file. Thin wrapper around the Honker SQLite loadable
   extension; no Redis, no external broker.
 executables: []
-extensions: []
+extensions:
+- ext/honker/extconf.rb
 extra_rdoc_files: []
 files:
+- LICENSE
+- LICENSE-APACHE
+- LICENSE-MIT
 - README.md
+- ext/honker/extconf.rb
+- ext/honker/honker-core/Cargo.toml
+- ext/honker/honker-core/LICENSE
+- ext/honker/honker-core/LICENSE-APACHE
+- ext/honker/honker-core/LICENSE-MIT
+- ext/honker/honker-core/README.md
+- ext/honker/honker-core/src/cron.rs
+- ext/honker/honker-core/src/honker_ops.rs
+- ext/honker/honker-core/src/kernel_watcher.rs
+- ext/honker/honker-core/src/lib.rs
+- ext/honker/honker-core/src/shm_watcher.rs
+- ext/honker/honker-extension/Cargo.toml
+- ext/honker/honker-extension/LICENSE
+- ext/honker/honker-extension/LICENSE-APACHE
+- ext/honker/honker-extension/LICENSE-MIT
+- ext/honker/honker-extension/README.md
+- ext/honker/honker-extension/src/lib.rs
 - honker.gemspec
 - lib/honker.rb
+- lib/honker/README.md
 - lib/honker/lock.rb
+- lib/honker/railtie.rb
 - lib/honker/scheduler.rb
 - lib/honker/stream.rb
 - lib/honker/transaction.rb
 - lib/honker/version.rb
 homepage: https://honker.dev
 licenses:
+- MIT
 - Apache-2.0
 metadata:
   homepage_uri: https://honker.dev
   source_code_uri: https://github.com/russellromney/honker
   documentation_uri: https://honker.dev/
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -61,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Durable queues, streams, pub/sub, and scheduler on SQLite.
 test_files: []