honker 0.1.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +193 -2
- data/ext/honker/extconf.rb +69 -0
- data/ext/honker/honker-core/Cargo.toml +49 -0
- data/ext/honker/honker-core/LICENSE +6 -0
- data/ext/honker/honker-core/LICENSE-APACHE +176 -0
- data/ext/honker/honker-core/LICENSE-MIT +21 -0
- data/ext/honker/honker-core/README.md +30 -0
- data/ext/honker/honker-core/src/cron.rs +473 -0
- data/ext/honker/honker-core/src/honker_ops.rs +1508 -0
- data/ext/honker/honker-core/src/kernel_watcher.rs +434 -0
- data/ext/honker/honker-core/src/lib.rs +3165 -0
- data/ext/honker/honker-core/src/shm_watcher.rs +250 -0
- data/ext/honker/honker-extension/Cargo.toml +36 -0
- data/ext/honker/honker-extension/LICENSE +6 -0
- data/ext/honker/honker-extension/LICENSE-APACHE +176 -0
- data/ext/honker/honker-extension/LICENSE-MIT +21 -0
- data/ext/honker/honker-extension/README.md +41 -0
- data/ext/honker/honker-extension/src/lib.rs +392 -0
- data/honker.gemspec +24 -1
- data/lib/honker/README.md +28 -0
- data/lib/honker/railtie.rb +11 -0
- data/lib/honker/scheduler.rb +59 -0
- data/lib/honker/version.rb +1 -1
- data/lib/honker.rb +125 -7
- metadata +39 -8
data/lib/honker.rb
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
require "json"
|
|
21
21
|
require "fiddle"
|
|
22
|
+
require "rbconfig"
|
|
22
23
|
require "sqlite3"
|
|
23
24
|
|
|
24
25
|
require_relative "honker/version"
|
|
@@ -26,14 +27,97 @@ require_relative "honker/transaction"
|
|
|
26
27
|
require_relative "honker/stream"
|
|
27
28
|
require_relative "honker/scheduler"
|
|
28
29
|
require_relative "honker/lock"
|
|
30
|
+
require_relative "honker/railtie" if defined?(::Rails::Railtie)
|
|
29
31
|
|
|
30
32
|
module Honker
|
|
33
|
+
# Honker's error class, raised by ExtensionResolver and CoreWatcher.
|
|
34
|
+
class Error < StandardError; end
|
|
35
|
+
|
|
36
|
+
# Resolves the path to the Honker SQLite loadable extension. Platform
|
|
37
|
+
# gems ship it bundled in lib/honker/; an explicit path and the
|
|
38
|
+
# HONKER_EXTENSION_PATH override take precedence.
|
|
39
|
+
class ExtensionResolver
|
|
40
|
+
def initialize(env: ENV.fetch("HONKER_EXTENSION_PATH", nil), bundled: nil)
|
|
41
|
+
@env = env
|
|
42
|
+
@bundled = bundled || File.expand_path("honker/#{extension_filename}", __dir__)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Returns the extension path: an explicit `extension_path`, else
|
|
46
|
+
# HONKER_EXTENSION_PATH, else the bundled extension. Raises
|
|
47
|
+
# Honker::Error when HONKER_EXTENSION_PATH or the bundled extension
|
|
48
|
+
# is missing.
|
|
49
|
+
def resolve(extension_path = nil)
|
|
50
|
+
return extension_path unless extension_path.nil?
|
|
51
|
+
return env_extension unless env.nil? || env.empty?
|
|
52
|
+
|
|
53
|
+
path = bundled
|
|
54
|
+
return path if File.file?(path)
|
|
55
|
+
|
|
56
|
+
raise Error, "Honker SQLite extension not found at #{path}; " \
|
|
57
|
+
"set HONKER_EXTENSION_PATH or pass extension_path:"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
attr_reader :env, :bundled
|
|
63
|
+
|
|
64
|
+
def env_extension
|
|
65
|
+
return env if File.file?(env)
|
|
66
|
+
|
|
67
|
+
raise Error, "HONKER_EXTENSION_PATH does not exist: #{env}"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def extension_filename
|
|
71
|
+
case RbConfig::CONFIG.fetch("host_os")
|
|
72
|
+
when /mswin|mingw|cygwin/ then "honker_ext.dll"
|
|
73
|
+
when /darwin/ then "libhonker_ext.dylib"
|
|
74
|
+
else "libhonker_ext.so"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Resolve the bundled (or overridden) extension path without naming
|
|
80
|
+
# ExtensionResolver — useful for `database.yml` ERB and tooling.
|
|
81
|
+
def self.extension_path(override = nil)
|
|
82
|
+
ExtensionResolver.new.resolve(override)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Load the Honker extension onto a raw SQLite3::Database. Encapsulates
|
|
86
|
+
# the enable_load_extension(true)/load/enable_load_extension(false)
|
|
87
|
+
# sequence so the toggle-off can't be forgotten.
|
|
88
|
+
def self.load_extension(sqlite_conn, extension_path: nil)
|
|
89
|
+
resolved = ExtensionResolver.new.resolve(extension_path)
|
|
90
|
+
sqlite_conn.enable_load_extension(true)
|
|
91
|
+
sqlite_conn.load_extension(resolved)
|
|
92
|
+
ensure
|
|
93
|
+
sqlite_conn.enable_load_extension(false)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Run honker_bootstrap() on the connection. Idempotent. Separated from
|
|
97
|
+
# load_extension so production users can opt to bootstrap from a
|
|
98
|
+
# migration instead of every connect.
|
|
99
|
+
def self.bootstrap(sqlite_conn)
|
|
100
|
+
sqlite_conn.execute("SELECT honker_bootstrap()")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Convenience: load the extension then bootstrap. The one-liner most
|
|
104
|
+
# ORM integrations reach for.
|
|
105
|
+
def self.setup(sqlite_conn, extension_path: nil, bootstrap: true)
|
|
106
|
+
load_extension(sqlite_conn, extension_path: extension_path)
|
|
107
|
+
self.bootstrap(sqlite_conn) if bootstrap
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Returns a Proc suitable for Sequel/Rom/Hanami `after_connect:`.
|
|
111
|
+
def self.sequel_after_connect(extension_path: nil, bootstrap: true)
|
|
112
|
+
proc { |conn| setup(conn, extension_path: extension_path, bootstrap: bootstrap) }
|
|
113
|
+
end
|
|
114
|
+
|
|
31
115
|
class CoreWatcher
|
|
32
|
-
def initialize(db_path, extension_path, backend)
|
|
116
|
+
def initialize(db_path, extension_path, backend, watcher_poll_interval_ms)
|
|
33
117
|
@lib = Fiddle.dlopen(extension_path)
|
|
34
118
|
@open = Fiddle::Function.new(
|
|
35
|
-
@lib["
|
|
36
|
-
[Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T],
|
|
119
|
+
@lib["honker_watcher_open_v2"],
|
|
120
|
+
[Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_LONG_LONG, Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T],
|
|
37
121
|
Fiddle::TYPE_VOIDP,
|
|
38
122
|
)
|
|
39
123
|
@wait = Fiddle::Function.new(
|
|
@@ -47,7 +131,13 @@ module Honker
|
|
|
47
131
|
Fiddle::TYPE_VOID,
|
|
48
132
|
)
|
|
49
133
|
err = "\0" * 1024
|
|
50
|
-
@handle = @open.call(
|
|
134
|
+
@handle = @open.call(
|
|
135
|
+
db_path.to_s,
|
|
136
|
+
backend.to_s,
|
|
137
|
+
watcher_poll_interval_ms || 1,
|
|
138
|
+
err,
|
|
139
|
+
err.bytesize,
|
|
140
|
+
)
|
|
51
141
|
return unless @handle.to_i.zero?
|
|
52
142
|
|
|
53
143
|
raise ArgumentError, err.delete_suffix("\0").split("\0", 2).first
|
|
@@ -86,21 +176,27 @@ module Honker
|
|
|
86
176
|
class Database
|
|
87
177
|
attr_reader :db
|
|
88
178
|
|
|
89
|
-
def initialize(path, extension_path
|
|
179
|
+
def initialize(path, extension_path: nil, watcher_backend: nil,
|
|
180
|
+
watcher_poll_interval_ms: nil,
|
|
181
|
+
extension_resolver: ExtensionResolver.new)
|
|
90
182
|
unless watcher_backend.nil? || watcher_backend.is_a?(String)
|
|
91
183
|
raise ArgumentError, "unknown watcher backend"
|
|
92
184
|
end
|
|
185
|
+
unless watcher_poll_interval_ms.nil? || watcher_poll_interval_ms.to_i.positive?
|
|
186
|
+
raise ArgumentError, "watcher_poll_interval_ms must be positive"
|
|
187
|
+
end
|
|
93
188
|
|
|
189
|
+
resolved_extension = extension_resolver.resolve(extension_path)
|
|
94
190
|
@db = SQLite3::Database.new(path)
|
|
95
191
|
@local_update_seq = 0
|
|
96
192
|
@db.busy_timeout = 5000
|
|
97
193
|
@db.execute("PRAGMA mmap_size = 0")
|
|
98
194
|
@db.enable_load_extension(true)
|
|
99
|
-
@db.load_extension(
|
|
195
|
+
@db.load_extension(resolved_extension)
|
|
100
196
|
@db.enable_load_extension(false)
|
|
101
197
|
@db.execute_batch(DEFAULT_PRAGMAS)
|
|
102
198
|
@db.execute("SELECT honker_bootstrap()")
|
|
103
|
-
@watcher = CoreWatcher.new(path,
|
|
199
|
+
@watcher = CoreWatcher.new(path, resolved_extension, watcher_backend, watcher_poll_interval_ms)
|
|
104
200
|
end
|
|
105
201
|
|
|
106
202
|
def close
|
|
@@ -348,6 +444,28 @@ module Honker
|
|
|
348
444
|
[job_id, worker_id, extend_s],
|
|
349
445
|
)[0] == 1
|
|
350
446
|
end
|
|
447
|
+
|
|
448
|
+
# Delete a pending or processing job by id. Returns true iff a row
|
|
449
|
+
# was removed. Idempotent on missing.
|
|
450
|
+
#
|
|
451
|
+
# IMPORTANT: cancel does NOT interrupt a worker currently running
|
|
452
|
+
# the handler. It invalidates the worker's claim — its next
|
|
453
|
+
# ack/heartbeat returns false. If you need the handler to actually
|
|
454
|
+
# halt, build that signal in your app.
|
|
455
|
+
def cancel(job_id)
|
|
456
|
+
n = @db.db.get_first_row("SELECT honker_cancel(?)", [job_id])[0]
|
|
457
|
+
@db.mark_updated if n.positive?
|
|
458
|
+
n.positive?
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# Read a single job row by id. Returns a Hash with the row fields,
|
|
462
|
+
# or nil if the job has been ack'd, dead'd, or never existed.
|
|
463
|
+
def get_job(job_id)
|
|
464
|
+
raw = @db.db.get_first_row("SELECT honker_get_job(?)", [job_id])[0]
|
|
465
|
+
return nil if raw.nil? || raw.empty?
|
|
466
|
+
|
|
467
|
+
JSON.parse(raw)
|
|
468
|
+
end
|
|
351
469
|
end
|
|
352
470
|
|
|
353
471
|
class Outbox
|
metadata
CHANGED
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: honker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Russell Romney
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: fiddle
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '1.0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '1.0'
|
|
13
26
|
- !ruby/object:Gem::Dependency
|
|
14
27
|
name: sqlite3
|
|
15
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -35,18 +48,37 @@ description: |-
|
|
|
35
48
|
streams, pub/sub, time-trigger scheduler, results, locks, rate limits, all
|
|
36
49
|
in one .db file. Thin wrapper around the Honker SQLite loadable
|
|
37
50
|
extension; no Redis, no external broker.
|
|
38
|
-
email:
|
|
39
51
|
executables: []
|
|
40
|
-
extensions:
|
|
52
|
+
extensions:
|
|
53
|
+
- ext/honker/extconf.rb
|
|
41
54
|
extra_rdoc_files: []
|
|
42
55
|
files:
|
|
43
56
|
- LICENSE
|
|
44
57
|
- LICENSE-APACHE
|
|
45
58
|
- LICENSE-MIT
|
|
46
59
|
- README.md
|
|
60
|
+
- ext/honker/extconf.rb
|
|
61
|
+
- ext/honker/honker-core/Cargo.toml
|
|
62
|
+
- ext/honker/honker-core/LICENSE
|
|
63
|
+
- ext/honker/honker-core/LICENSE-APACHE
|
|
64
|
+
- ext/honker/honker-core/LICENSE-MIT
|
|
65
|
+
- ext/honker/honker-core/README.md
|
|
66
|
+
- ext/honker/honker-core/src/cron.rs
|
|
67
|
+
- ext/honker/honker-core/src/honker_ops.rs
|
|
68
|
+
- ext/honker/honker-core/src/kernel_watcher.rs
|
|
69
|
+
- ext/honker/honker-core/src/lib.rs
|
|
70
|
+
- ext/honker/honker-core/src/shm_watcher.rs
|
|
71
|
+
- ext/honker/honker-extension/Cargo.toml
|
|
72
|
+
- ext/honker/honker-extension/LICENSE
|
|
73
|
+
- ext/honker/honker-extension/LICENSE-APACHE
|
|
74
|
+
- ext/honker/honker-extension/LICENSE-MIT
|
|
75
|
+
- ext/honker/honker-extension/README.md
|
|
76
|
+
- ext/honker/honker-extension/src/lib.rs
|
|
47
77
|
- honker.gemspec
|
|
48
78
|
- lib/honker.rb
|
|
79
|
+
- lib/honker/README.md
|
|
49
80
|
- lib/honker/lock.rb
|
|
81
|
+
- lib/honker/railtie.rb
|
|
50
82
|
- lib/honker/scheduler.rb
|
|
51
83
|
- lib/honker/stream.rb
|
|
52
84
|
- lib/honker/transaction.rb
|
|
@@ -59,7 +91,7 @@ metadata:
|
|
|
59
91
|
homepage_uri: https://honker.dev
|
|
60
92
|
source_code_uri: https://github.com/russellromney/honker
|
|
61
93
|
documentation_uri: https://honker.dev/
|
|
62
|
-
|
|
94
|
+
rubygems_mfa_required: 'true'
|
|
63
95
|
rdoc_options: []
|
|
64
96
|
require_paths:
|
|
65
97
|
- lib
|
|
@@ -74,8 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
74
106
|
- !ruby/object:Gem::Version
|
|
75
107
|
version: '0'
|
|
76
108
|
requirements: []
|
|
77
|
-
rubygems_version: 3.
|
|
78
|
-
signing_key:
|
|
109
|
+
rubygems_version: 3.6.9
|
|
79
110
|
specification_version: 4
|
|
80
111
|
summary: Durable queues, streams, pub/sub, and scheduler on SQLite.
|
|
81
112
|
test_files: []
|