pakyow-data 1.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +0 -0
- data/LICENSE +4 -0
- data/README.md +29 -0
- data/lib/pakyow/data/adapters/abstract.rb +58 -0
- data/lib/pakyow/data/adapters/sql/commands.rb +58 -0
- data/lib/pakyow/data/adapters/sql/dataset_methods.rb +29 -0
- data/lib/pakyow/data/adapters/sql/differ.rb +76 -0
- data/lib/pakyow/data/adapters/sql/migrator/adapter_methods.rb +95 -0
- data/lib/pakyow/data/adapters/sql/migrator.rb +181 -0
- data/lib/pakyow/data/adapters/sql/migrators/automator.rb +49 -0
- data/lib/pakyow/data/adapters/sql/migrators/finalizer.rb +96 -0
- data/lib/pakyow/data/adapters/sql/runner.rb +49 -0
- data/lib/pakyow/data/adapters/sql/source_extension.rb +31 -0
- data/lib/pakyow/data/adapters/sql/types.rb +50 -0
- data/lib/pakyow/data/adapters/sql.rb +247 -0
- data/lib/pakyow/data/behavior/config.rb +28 -0
- data/lib/pakyow/data/behavior/lookup.rb +75 -0
- data/lib/pakyow/data/behavior/serialization.rb +40 -0
- data/lib/pakyow/data/connection.rb +103 -0
- data/lib/pakyow/data/container.rb +273 -0
- data/lib/pakyow/data/errors.rb +169 -0
- data/lib/pakyow/data/framework.rb +42 -0
- data/lib/pakyow/data/helpers.rb +11 -0
- data/lib/pakyow/data/lookup.rb +85 -0
- data/lib/pakyow/data/migrator.rb +182 -0
- data/lib/pakyow/data/object.rb +98 -0
- data/lib/pakyow/data/proxy.rb +262 -0
- data/lib/pakyow/data/result.rb +53 -0
- data/lib/pakyow/data/sources/abstract.rb +82 -0
- data/lib/pakyow/data/sources/ephemeral.rb +72 -0
- data/lib/pakyow/data/sources/relational/association.rb +43 -0
- data/lib/pakyow/data/sources/relational/associations/belongs_to.rb +47 -0
- data/lib/pakyow/data/sources/relational/associations/has_many.rb +54 -0
- data/lib/pakyow/data/sources/relational/associations/has_one.rb +54 -0
- data/lib/pakyow/data/sources/relational/associations/through.rb +67 -0
- data/lib/pakyow/data/sources/relational/command.rb +531 -0
- data/lib/pakyow/data/sources/relational/migrator.rb +101 -0
- data/lib/pakyow/data/sources/relational.rb +587 -0
- data/lib/pakyow/data/subscribers/adapters/memory.rb +153 -0
- data/lib/pakyow/data/subscribers/adapters/redis/pipeliner.rb +45 -0
- data/lib/pakyow/data/subscribers/adapters/redis/scripts/_shared.lua +73 -0
- data/lib/pakyow/data/subscribers/adapters/redis/scripts/expire.lua +16 -0
- data/lib/pakyow/data/subscribers/adapters/redis/scripts/persist.lua +15 -0
- data/lib/pakyow/data/subscribers/adapters/redis/scripts/register.lua +37 -0
- data/lib/pakyow/data/subscribers/adapters/redis.rb +209 -0
- data/lib/pakyow/data/subscribers.rb +148 -0
- data/lib/pakyow/data/tasks/bootstrap.rake +18 -0
- data/lib/pakyow/data/tasks/create.rake +22 -0
- data/lib/pakyow/data/tasks/drop.rake +32 -0
- data/lib/pakyow/data/tasks/finalize.rake +56 -0
- data/lib/pakyow/data/tasks/migrate.rake +24 -0
- data/lib/pakyow/data/tasks/reset.rake +18 -0
- data/lib/pakyow/data/types.rb +37 -0
- data/lib/pakyow/data.rb +27 -0
- data/lib/pakyow/environment/data/auto_migrate.rb +31 -0
- data/lib/pakyow/environment/data/config.rb +54 -0
- data/lib/pakyow/environment/data/connections.rb +76 -0
- data/lib/pakyow/environment/data/memory_db.rb +23 -0
- data/lib/pakyow/validations/unique.rb +26 -0
- metadata +186 -0
@@ -0,0 +1,153 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
require "concurrent/array"
|
6
|
+
require "concurrent/hash"
|
7
|
+
require "concurrent/timer_task"
|
8
|
+
|
9
|
+
require "pakyow/support/deep_dup"
|
10
|
+
require "pakyow/support/deep_freeze"
|
11
|
+
|
12
|
+
module Pakyow
|
13
|
+
module Data
|
14
|
+
class Subscribers
|
15
|
+
module Adapters
|
16
|
+
# Manages data subscriptions in memory.
|
17
|
+
#
|
18
|
+
# Great for development, not for use in production!
|
19
|
+
#
|
20
|
+
# @api private
|
21
|
+
class Memory
|
22
|
+
class << self
|
23
|
+
def generate_subscription_id(subscription)
|
24
|
+
Digest::SHA1.hexdigest(Marshal.dump(subscription))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
using Support::DeepDup
|
29
|
+
|
30
|
+
extend Support::DeepFreeze
|
31
|
+
unfreezable :subscriptions_by_id, :subscription_ids_by_source, :subscribers_by_subscription_id, :subscription_ids_by_subscriber, :expirations_for_subscriber
|
32
|
+
|
33
|
+
def initialize(*)
|
34
|
+
@subscriptions_by_id = Concurrent::Hash.new
|
35
|
+
@subscription_ids_by_source = Concurrent::Hash.new
|
36
|
+
@subscribers_by_subscription_id = Concurrent::Hash.new
|
37
|
+
@subscription_ids_by_subscriber = Concurrent::Hash.new
|
38
|
+
@expirations_for_subscriber = Concurrent::Hash.new
|
39
|
+
|
40
|
+
Concurrent::TimerTask.new(execution_interval: 10, timeout_interval: 10) {
|
41
|
+
@expirations_for_subscriber.each do |subscriber, timeout|
|
42
|
+
if timeout < Time.now
|
43
|
+
unsubscribe(subscriber)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
}.execute
|
47
|
+
end
|
48
|
+
|
49
|
+
def register_subscriptions(subscriptions, subscriber: nil)
|
50
|
+
subscriptions.map { |subscription|
|
51
|
+
self.class.generate_subscription_id(subscription).tap do |subscription_id|
|
52
|
+
register_subscription_with_subscription_id(subscription, subscription_id)
|
53
|
+
register_subscription_id_for_source(subscription_id, subscription[:source])
|
54
|
+
register_subscriber_for_subscription_id(subscriber, subscription_id)
|
55
|
+
end
|
56
|
+
}
|
57
|
+
end
|
58
|
+
|
59
|
+
def subscriptions_for_source(source)
|
60
|
+
subscription_ids_for_source(source).map { |subscription_id|
|
61
|
+
subscription_with_id(subscription_id)
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
def unsubscribe(subscriber)
|
66
|
+
subscription_ids_for_subscriber(subscriber).dup.each do |subscription_id|
|
67
|
+
unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def expire(subscriber, seconds)
|
72
|
+
@expirations_for_subscriber[subscriber] = Time.now + seconds
|
73
|
+
end
|
74
|
+
|
75
|
+
def persist(subscriber)
|
76
|
+
@expirations_for_subscriber.delete(subscriber)
|
77
|
+
end
|
78
|
+
|
79
|
+
def expiring?(subscriber)
|
80
|
+
@expirations_for_subscriber.key?(subscriber)
|
81
|
+
end
|
82
|
+
|
83
|
+
def subscribers_for_subscription_id(subscription_id)
|
84
|
+
@subscribers_by_subscription_id[subscription_id] || []
|
85
|
+
end
|
86
|
+
|
87
|
+
SERIALIZABLE_IVARS = %i(
|
88
|
+
@subscriptions_by_id
|
89
|
+
@subscription_ids_by_source
|
90
|
+
@subscribers_by_subscription_id
|
91
|
+
@expirations_for_subscriber
|
92
|
+
).freeze
|
93
|
+
|
94
|
+
def serialize
|
95
|
+
SERIALIZABLE_IVARS.each_with_object({}) do |ivar, hash|
|
96
|
+
hash[ivar] = instance_variable_get(ivar)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
protected
|
101
|
+
|
102
|
+
def subscription_ids_for_source(source)
|
103
|
+
(@subscription_ids_by_source[source] || []).select { |subscription_id|
|
104
|
+
subscribers_for_subscription_id(subscription_id).any? { |subscriber|
|
105
|
+
!expiring?(subscriber)
|
106
|
+
}
|
107
|
+
}
|
108
|
+
end
|
109
|
+
|
110
|
+
def subscription_with_id(subscription_id)
|
111
|
+
subscription = @subscriptions_by_id[subscription_id].deep_dup
|
112
|
+
subscription[:id] = subscription_id
|
113
|
+
subscription
|
114
|
+
end
|
115
|
+
|
116
|
+
def subscription_ids_for_subscriber(subscriber)
|
117
|
+
@subscription_ids_by_subscriber[subscriber] || []
|
118
|
+
end
|
119
|
+
|
120
|
+
def register_subscription_with_subscription_id(subscription, subscription_id)
|
121
|
+
@subscriptions_by_id[subscription_id] = subscription
|
122
|
+
end
|
123
|
+
|
124
|
+
def register_subscription_id_for_source(subscription_id, source)
|
125
|
+
@subscription_ids_by_source[source] ||= Concurrent::Array.new
|
126
|
+
(@subscription_ids_by_source[source] << subscription_id).uniq!
|
127
|
+
end
|
128
|
+
|
129
|
+
def register_subscriber_for_subscription_id(subscriber, subscription_id)
|
130
|
+
@subscribers_by_subscription_id[subscription_id] ||= Concurrent::Array.new
|
131
|
+
(@subscribers_by_subscription_id[subscription_id] << subscriber).uniq!
|
132
|
+
|
133
|
+
@subscription_ids_by_subscriber[subscriber] ||= Concurrent::Array.new
|
134
|
+
(@subscription_ids_by_subscriber[subscriber] << subscription_id).uniq!
|
135
|
+
end
|
136
|
+
|
137
|
+
def unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
|
138
|
+
subscribers_for_subscription_id(subscription_id).delete(subscriber)
|
139
|
+
subscription_ids_for_subscriber(subscriber).delete(subscription_id)
|
140
|
+
|
141
|
+
if subscribers_for_subscription_id(subscription_id).empty?
|
142
|
+
@subscriptions_by_id.delete(subscription_id)
|
143
|
+
|
144
|
+
@subscription_ids_by_source.each do |_, ids|
|
145
|
+
ids.delete(subscription_id)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Pakyow
|
4
|
+
module Data
|
5
|
+
class Subscribers
|
6
|
+
module Adapters
|
7
|
+
class Redis
|
8
|
+
# @api private
|
9
|
+
class Pipeliner
|
10
|
+
TIMEOUT = 1.0 / 100.0
|
11
|
+
|
12
|
+
def initialize(redis)
|
13
|
+
@redis = redis
|
14
|
+
@commands = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def enqueue(future)
|
18
|
+
@commands << { future: future, callback: Proc.new }
|
19
|
+
end
|
20
|
+
|
21
|
+
def wait
|
22
|
+
@commands.map { |command|
|
23
|
+
while command[:future].value.is_a?(::Redis::FutureNotReady)
|
24
|
+
sleep TIMEOUT
|
25
|
+
end
|
26
|
+
|
27
|
+
command[:callback].call(command[:future].value)
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.pipeline(redis)
|
32
|
+
pipeliner = Pipeliner.new(redis)
|
33
|
+
|
34
|
+
redis.pipelined do
|
35
|
+
yield pipeliner
|
36
|
+
end
|
37
|
+
|
38
|
+
pipeliner.wait
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
local function massive_redis_command(command, key, t)
|
2
|
+
local i = 1
|
3
|
+
local temp = {}
|
4
|
+
|
5
|
+
while(i <= #t) do
|
6
|
+
table.insert(temp, t[i + 1])
|
7
|
+
table.insert(temp, t[i])
|
8
|
+
|
9
|
+
if #temp >= 1000 then
|
10
|
+
redis.call(command, key, unpack(temp))
|
11
|
+
temp = {}
|
12
|
+
end
|
13
|
+
|
14
|
+
i = i + 2
|
15
|
+
end
|
16
|
+
|
17
|
+
if #temp > 0 then
|
18
|
+
redis.call(command, key, unpack(temp))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
local function key_subscription_id(key_prefix, key_delimeter, subscription_id)
|
23
|
+
return key_prefix .. key_delimeter .. "subscription:" .. subscription_id
|
24
|
+
end
|
25
|
+
|
26
|
+
local function key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
|
27
|
+
return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "subscribers"
|
28
|
+
end
|
29
|
+
|
30
|
+
local function key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id)
|
31
|
+
return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "source"
|
32
|
+
end
|
33
|
+
|
34
|
+
local function key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber)
|
35
|
+
return key_prefix .. key_delimeter .. "subscriber:" .. subscriber
|
36
|
+
end
|
37
|
+
|
38
|
+
local function key_subscription_ids_by_source(key_prefix, key_delimeter, source)
|
39
|
+
return key_prefix .. key_delimeter .. "source:" .. source
|
40
|
+
end
|
41
|
+
|
42
|
+
local function subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, min, max)
|
43
|
+
return redis.call("zrangebyscore", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), min, max)
|
44
|
+
end
|
45
|
+
|
46
|
+
local function expire_subscription(key_prefix, key_delimeter, subscription_id)
|
47
|
+
local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
|
48
|
+
|
49
|
+
-- only expire the subscription if it is not related to a non-expiring subscriber
|
50
|
+
if redis.call("zcount", subscription_key, "+inf", "+inf") == 0 then
|
51
|
+
local time_expire = redis.call("zrevrangebyscore", subscription_key, "+inf", 0, "WITHSCORES", "LIMIT", 0, 1)[2]
|
52
|
+
|
53
|
+
if time_expire ~= "inf" then
|
54
|
+
local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
|
55
|
+
redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), time_expire, subscription_id)
|
56
|
+
|
57
|
+
redis.call("expireat", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
|
58
|
+
redis.call("expireat", subscription_key, time_expire + 1)
|
59
|
+
redis.call("expireat", key_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
local function persist_subscription(key_prefix, key_delimeter, subscription_id)
|
65
|
+
local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
|
66
|
+
|
67
|
+
local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
|
68
|
+
redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
|
69
|
+
|
70
|
+
redis.call("persist", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
|
71
|
+
redis.call("persist", subscription_key)
|
72
|
+
redis.call("persist", key_subscription_id(key_prefix, key_delimeter, subscription_id))
|
73
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
local key_prefix = ARGV[1]
|
2
|
+
local key_delimeter = ARGV[2]
|
3
|
+
local subscriber = ARGV[3]
|
4
|
+
local time_expire = ARGV[4]
|
5
|
+
|
6
|
+
redis.call("expireat", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), time_expire + 1)
|
7
|
+
|
8
|
+
local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, "+inf", "+inf")
|
9
|
+
|
10
|
+
local i = 1
|
11
|
+
while(i <= #subscription_ids) do
|
12
|
+
local subscription_id = subscription_ids[i]
|
13
|
+
redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire, subscriber)
|
14
|
+
expire_subscription(key_prefix, key_delimeter, subscription_id)
|
15
|
+
i = i + 1
|
16
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
local key_prefix = ARGV[1]
|
2
|
+
local key_delimeter = ARGV[2]
|
3
|
+
local subscriber = ARGV[3]
|
4
|
+
|
5
|
+
redis.call("persist", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
|
6
|
+
|
7
|
+
local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, 0, "+inf")
|
8
|
+
|
9
|
+
local i = 1
|
10
|
+
while(i <= #subscription_ids) do
|
11
|
+
local subscription_id = subscription_ids[i]
|
12
|
+
redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), "+inf", subscriber)
|
13
|
+
persist_subscription(key_prefix, key_delimeter, subscription_id)
|
14
|
+
i = i + 1
|
15
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
local key_prefix = ARGV[1]
|
2
|
+
local key_delimeter = ARGV[2]
|
3
|
+
local subscriber = ARGV[3]
|
4
|
+
local subscription_id = ARGV[4]
|
5
|
+
local subscription_string = ARGV[5]
|
6
|
+
local source = ARGV[6]
|
7
|
+
local now = ARGV[7]
|
8
|
+
|
9
|
+
local expiry = "+inf"
|
10
|
+
|
11
|
+
-- determine if the subscriber is expiring
|
12
|
+
local ttl = redis.call("ttl", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
|
13
|
+
|
14
|
+
if ttl > -1 then
|
15
|
+
expiry = now + ttl
|
16
|
+
end
|
17
|
+
|
18
|
+
-- store the subscription
|
19
|
+
redis.call("set", key_subscription_id(key_prefix, key_delimeter, subscription_id), subscription_string)
|
20
|
+
|
21
|
+
-- add the subscription to the subscriber's set
|
22
|
+
redis.call("zadd", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), expiry, subscription_id)
|
23
|
+
|
24
|
+
-- add the subscriber to the subscription's set
|
25
|
+
redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), expiry, subscriber)
|
26
|
+
|
27
|
+
-- add the subscription to the source's set
|
28
|
+
redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
|
29
|
+
|
30
|
+
-- define what source the subscription is for
|
31
|
+
redis.call("set", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), source)
|
32
|
+
|
33
|
+
if ttl > -1 then
|
34
|
+
expire_subscription(key_prefix, key_delimeter, subscription_id)
|
35
|
+
else
|
36
|
+
persist_subscription(key_prefix, key_delimeter, subscription_id)
|
37
|
+
end
|
@@ -0,0 +1,209 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
require "redis"
|
6
|
+
require "concurrent/timer_task"
|
7
|
+
require "connection_pool"
|
8
|
+
|
9
|
+
require "pakyow/support/deep_freeze"
|
10
|
+
|
11
|
+
module Pakyow
|
12
|
+
module Data
|
13
|
+
class Subscribers
|
14
|
+
module Adapters
|
15
|
+
# Manages data subscriptions in redis.
|
16
|
+
#
|
17
|
+
# Use this in production.
|
18
|
+
#
|
19
|
+
# @api private
|
20
|
+
class Redis
|
21
|
+
class << self
|
22
|
+
def stringify_subscription(subscription)
|
23
|
+
Marshal.dump(subscription)
|
24
|
+
end
|
25
|
+
|
26
|
+
def generate_subscription_id(subscription_string)
|
27
|
+
Digest::SHA1.hexdigest(subscription_string)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
SCRIPTS = %i(register expire persist).freeze
|
32
|
+
KEY_PART_SEPARATOR = "/"
|
33
|
+
KEY_PREFIX = "data"
|
34
|
+
INFINITY = "+inf"
|
35
|
+
|
36
|
+
extend Support::DeepFreeze
|
37
|
+
unfreezable :redis
|
38
|
+
|
39
|
+
def initialize(config)
|
40
|
+
@redis = ConnectionPool.new(**config[:pool]) {
|
41
|
+
::Redis.new(config[:connection])
|
42
|
+
}
|
43
|
+
|
44
|
+
@prefix = [config[:key_prefix], KEY_PREFIX].join(KEY_PART_SEPARATOR)
|
45
|
+
|
46
|
+
@scripts = {}
|
47
|
+
load_scripts
|
48
|
+
|
49
|
+
Concurrent::TimerTask.new(execution_interval: 300, timeout_interval: 300) {
|
50
|
+
cleanup
|
51
|
+
}.execute
|
52
|
+
end
|
53
|
+
|
54
|
+
def register_subscriptions(subscriptions, subscriber:)
|
55
|
+
[].tap do |subscription_ids|
|
56
|
+
subscriptions.each do |subscription|
|
57
|
+
subscription_string = self.class.stringify_subscription(subscription)
|
58
|
+
subscription_id = self.class.generate_subscription_id(subscription_string)
|
59
|
+
source = subscription[:source]
|
60
|
+
|
61
|
+
@redis.with do |redis|
|
62
|
+
redis.evalsha(@scripts[:register], argv: [
|
63
|
+
@prefix,
|
64
|
+
KEY_PART_SEPARATOR,
|
65
|
+
subscriber.to_s,
|
66
|
+
subscription_id,
|
67
|
+
subscription_string,
|
68
|
+
source.to_s,
|
69
|
+
Time.now.to_i
|
70
|
+
])
|
71
|
+
end
|
72
|
+
|
73
|
+
subscription_ids << subscription_id
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def subscriptions_for_source(source)
|
79
|
+
subscriptions_for_subscription_ids(subscription_ids_for_source(source)).compact
|
80
|
+
end
|
81
|
+
|
82
|
+
def unsubscribe(subscriber)
|
83
|
+
expire(subscriber, 0)
|
84
|
+
end
|
85
|
+
|
86
|
+
def expire(subscriber, seconds)
|
87
|
+
@redis.with do |redis|
|
88
|
+
redis.evalsha(@scripts[:expire], argv: [
|
89
|
+
@prefix,
|
90
|
+
KEY_PART_SEPARATOR,
|
91
|
+
subscriber.to_s,
|
92
|
+
Time.now.to_i + seconds
|
93
|
+
])
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def persist(subscriber)
|
98
|
+
@redis.with do |redis|
|
99
|
+
redis.evalsha(@scripts[:persist], argv: [
|
100
|
+
@prefix,
|
101
|
+
KEY_PART_SEPARATOR,
|
102
|
+
subscriber.to_s
|
103
|
+
])
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def expiring?(subscriber)
|
108
|
+
@redis.with do |redis|
|
109
|
+
redis.ttl(key_subscription_ids_by_subscriber(subscriber)) > -1
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def subscribers_for_subscription_id(subscription_id)
|
114
|
+
@redis.with do |redis|
|
115
|
+
redis.zrangebyscore(
|
116
|
+
key_subscribers_by_subscription_id(
|
117
|
+
subscription_id
|
118
|
+
), INFINITY, INFINITY
|
119
|
+
).map(&:to_sym)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def subscription_ids_for_source(source)
|
124
|
+
@redis.with do |redis|
|
125
|
+
redis.zrangebyscore(
|
126
|
+
key_subscription_ids_by_source(
|
127
|
+
source
|
128
|
+
), INFINITY, INFINITY
|
129
|
+
)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# FIXME: Refactor this into a lua script. We'll want to stop using SCAN and instead store
|
134
|
+
# known sources in a set. Cleanup should then be based off the set of known sources and
|
135
|
+
# return the number of values that were removed.
|
136
|
+
#
|
137
|
+
def cleanup
|
138
|
+
@redis.with do |redis|
|
139
|
+
redis.scan_each(match: key_subscription_ids_by_source("*")) do |key|
|
140
|
+
Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Cleaning up expired subscriptions for #{key}"
|
141
|
+
removed_count = redis.zremrangebyscore(key, 0, Time.now.to_i)
|
142
|
+
Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Removed #{removed_count} members for #{key}"
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
private
|
148
|
+
|
149
|
+
def subscriptions_for_subscription_ids(subscription_ids)
|
150
|
+
return [] if subscription_ids.empty?
|
151
|
+
|
152
|
+
@redis.with do |redis|
|
153
|
+
redis.mget(subscription_ids.map { |subscription_id|
|
154
|
+
key_subscription_id(subscription_id)
|
155
|
+
}).zip(subscription_ids).map { |subscription_string, subscription_id|
|
156
|
+
begin
|
157
|
+
Marshal.restore(subscription_string).tap do |subscription|
|
158
|
+
subscription[:id] = subscription_id
|
159
|
+
end
|
160
|
+
rescue TypeError
|
161
|
+
Pakyow.logger.error "could not find subscription for #{subscription_id}"
|
162
|
+
{}
|
163
|
+
end
|
164
|
+
}
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def build_key(*parts)
|
169
|
+
[@prefix].concat(parts).join(KEY_PART_SEPARATOR)
|
170
|
+
end
|
171
|
+
|
172
|
+
def key_subscription_id(subscription_id)
|
173
|
+
build_key("subscription:#{subscription_id}")
|
174
|
+
end
|
175
|
+
|
176
|
+
def key_subscribers_by_subscription_id(subscription_id)
|
177
|
+
build_key("subscription:#{subscription_id}", "subscribers")
|
178
|
+
end
|
179
|
+
|
180
|
+
def key_subscription_ids_by_subscriber(subscriber)
|
181
|
+
build_key("subscriber:#{subscriber}")
|
182
|
+
end
|
183
|
+
|
184
|
+
def key_subscription_ids_by_source(source)
|
185
|
+
build_key("source:#{source}")
|
186
|
+
end
|
187
|
+
|
188
|
+
def key_source_for_subscription_id(subscription_id)
|
189
|
+
build_key("subscription:#{subscription_id}", "source")
|
190
|
+
end
|
191
|
+
|
192
|
+
def load_scripts
|
193
|
+
@redis.with do |redis|
|
194
|
+
SCRIPTS.each do |script|
|
195
|
+
script_content = File.read(
|
196
|
+
File.expand_path("../redis/scripts/_shared.lua", __FILE__)
|
197
|
+
) + File.read(
|
198
|
+
File.expand_path("../redis/scripts/#{script}.lua", __FILE__)
|
199
|
+
)
|
200
|
+
|
201
|
+
@scripts[script] = redis.script(:load, script_content)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent/executor/thread_pool_executor"
|
4
|
+
|
5
|
+
require "pakyow/support/core_refinements/method/introspection"
|
6
|
+
require "pakyow/support/deep_freeze"
|
7
|
+
|
8
|
+
module Pakyow
|
9
|
+
module Data
|
10
|
+
# @api private
|
11
|
+
class Subscribers
|
12
|
+
attr_accessor :lookup, :adapter
|
13
|
+
|
14
|
+
extend Support::DeepFreeze
|
15
|
+
unfreezable :executor
|
16
|
+
|
17
|
+
using Support::Refinements::Method::Introspection
|
18
|
+
|
19
|
+
def initialize(app, adapter = :memory, adapter_config = {})
|
20
|
+
@app = app
|
21
|
+
|
22
|
+
require "pakyow/data/subscribers/adapters/#{adapter}"
|
23
|
+
@adapter = Pakyow::Data::Subscribers::Adapters.const_get(
|
24
|
+
adapter.to_s.capitalize
|
25
|
+
).new(
|
26
|
+
adapter_config.to_h.merge(
|
27
|
+
app.config.data.subscriptions.adapter_settings.to_h
|
28
|
+
)
|
29
|
+
)
|
30
|
+
|
31
|
+
@executor = Concurrent::ThreadPoolExecutor.new(
|
32
|
+
auto_terminate: false,
|
33
|
+
min_threads: 1,
|
34
|
+
max_threads: 10,
|
35
|
+
max_queue: 0
|
36
|
+
)
|
37
|
+
rescue LoadError, NameError => error
|
38
|
+
raise UnknownSubscriberAdapter.build(error, adapter: adapter)
|
39
|
+
end
|
40
|
+
|
41
|
+
def shutdown
|
42
|
+
@executor.shutdown
|
43
|
+
@executor.wait_for_termination(30)
|
44
|
+
end
|
45
|
+
|
46
|
+
def register_subscriptions(subscriptions, subscriber: nil, &block)
|
47
|
+
@executor << Proc.new {
|
48
|
+
subscriptions.each do |subscription|
|
49
|
+
subscription[:version] = @app.config.data.subscriptions.version
|
50
|
+
end
|
51
|
+
|
52
|
+
@adapter.register_subscriptions(subscriptions, subscriber: subscriber).tap do |ids|
|
53
|
+
yield ids if block_given?
|
54
|
+
end
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
def did_mutate(source_name, changed_values = nil, result_source = nil)
|
59
|
+
@executor << Proc.new {
|
60
|
+
begin
|
61
|
+
@adapter.subscriptions_for_source(source_name).select { |subscription|
|
62
|
+
process?(subscription, changed_values, result_source)
|
63
|
+
}.uniq.each do |subscription|
|
64
|
+
if subscription[:version] == @app.config.data.subscriptions.version
|
65
|
+
process(subscription, result_source)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
rescue => error
|
69
|
+
Pakyow.logger.error "[Pakyow::Data::Subscribers] did_mutate failed: #{error}"
|
70
|
+
end
|
71
|
+
}
|
72
|
+
end
|
73
|
+
|
74
|
+
def unsubscribe(subscriber)
|
75
|
+
@adapter.unsubscribe(subscriber)
|
76
|
+
end
|
77
|
+
|
78
|
+
def expire(subscriber, seconds)
|
79
|
+
@adapter.expire(subscriber, seconds)
|
80
|
+
end
|
81
|
+
|
82
|
+
def persist(subscriber)
|
83
|
+
@adapter.persist(subscriber)
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
def process?(subscription, changed_values, result_source)
|
89
|
+
subscription[:handler] && qualified_subscription?(subscription, changed_values, result_source)
|
90
|
+
end
|
91
|
+
|
92
|
+
def process(subscription, mutated_source)
|
93
|
+
callback = subscription[:handler].new(@app)
|
94
|
+
arguments = {}
|
95
|
+
|
96
|
+
if callback.method(:call).keyword_argument?(:id)
|
97
|
+
arguments[:id] = subscription[:id]
|
98
|
+
end
|
99
|
+
|
100
|
+
if callback.method(:call).keyword_argument?(:result)
|
101
|
+
arguments[:result] = if subscription[:ephemeral]
|
102
|
+
mutated_source
|
103
|
+
else
|
104
|
+
subscription[:proxy]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
if callback.method(:call).keyword_argument?(:subscription)
|
109
|
+
arguments[:subscription] = subscription
|
110
|
+
end
|
111
|
+
|
112
|
+
callback.call(subscription[:payload], **arguments)
|
113
|
+
end
|
114
|
+
|
115
|
+
def qualified_subscription?(subscription, changed_values, result_source)
|
116
|
+
if subscription[:ephemeral]
|
117
|
+
result_source.qualifications == subscription[:qualifications]
|
118
|
+
else
|
119
|
+
original_results = if result_source
|
120
|
+
result_source.original_results
|
121
|
+
else
|
122
|
+
[]
|
123
|
+
end
|
124
|
+
|
125
|
+
qualified?(
|
126
|
+
subscription.delete(:qualifications).to_a,
|
127
|
+
changed_values,
|
128
|
+
result_source.to_a,
|
129
|
+
original_results.to_a
|
130
|
+
)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
QUALIFIABLE_TYPES = [Hash, Support::IndifferentHash]
|
135
|
+
def qualified?(qualifications, changed_values, changed_results, original_results)
|
136
|
+
qualifications.all? do |key, value|
|
137
|
+
(QUALIFIABLE_TYPES.include?(changed_values.class) && changed_values.to_h[key] == value) || qualified_result?(key, value, changed_results, original_results)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def qualified_result?(key, value, changed_results, original_results)
|
142
|
+
original_results.concat(changed_results).any? do |result|
|
143
|
+
result[key] == value
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|