pakyow-data 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +0 -0
  3. data/LICENSE +4 -0
  4. data/README.md +29 -0
  5. data/lib/pakyow/data/adapters/abstract.rb +58 -0
  6. data/lib/pakyow/data/adapters/sql/commands.rb +58 -0
  7. data/lib/pakyow/data/adapters/sql/dataset_methods.rb +29 -0
  8. data/lib/pakyow/data/adapters/sql/differ.rb +76 -0
  9. data/lib/pakyow/data/adapters/sql/migrator/adapter_methods.rb +95 -0
  10. data/lib/pakyow/data/adapters/sql/migrator.rb +181 -0
  11. data/lib/pakyow/data/adapters/sql/migrators/automator.rb +49 -0
  12. data/lib/pakyow/data/adapters/sql/migrators/finalizer.rb +96 -0
  13. data/lib/pakyow/data/adapters/sql/runner.rb +49 -0
  14. data/lib/pakyow/data/adapters/sql/source_extension.rb +31 -0
  15. data/lib/pakyow/data/adapters/sql/types.rb +50 -0
  16. data/lib/pakyow/data/adapters/sql.rb +247 -0
  17. data/lib/pakyow/data/behavior/config.rb +28 -0
  18. data/lib/pakyow/data/behavior/lookup.rb +75 -0
  19. data/lib/pakyow/data/behavior/serialization.rb +40 -0
  20. data/lib/pakyow/data/connection.rb +103 -0
  21. data/lib/pakyow/data/container.rb +273 -0
  22. data/lib/pakyow/data/errors.rb +169 -0
  23. data/lib/pakyow/data/framework.rb +42 -0
  24. data/lib/pakyow/data/helpers.rb +11 -0
  25. data/lib/pakyow/data/lookup.rb +85 -0
  26. data/lib/pakyow/data/migrator.rb +182 -0
  27. data/lib/pakyow/data/object.rb +98 -0
  28. data/lib/pakyow/data/proxy.rb +262 -0
  29. data/lib/pakyow/data/result.rb +53 -0
  30. data/lib/pakyow/data/sources/abstract.rb +82 -0
  31. data/lib/pakyow/data/sources/ephemeral.rb +72 -0
  32. data/lib/pakyow/data/sources/relational/association.rb +43 -0
  33. data/lib/pakyow/data/sources/relational/associations/belongs_to.rb +47 -0
  34. data/lib/pakyow/data/sources/relational/associations/has_many.rb +54 -0
  35. data/lib/pakyow/data/sources/relational/associations/has_one.rb +54 -0
  36. data/lib/pakyow/data/sources/relational/associations/through.rb +67 -0
  37. data/lib/pakyow/data/sources/relational/command.rb +531 -0
  38. data/lib/pakyow/data/sources/relational/migrator.rb +101 -0
  39. data/lib/pakyow/data/sources/relational.rb +587 -0
  40. data/lib/pakyow/data/subscribers/adapters/memory.rb +153 -0
  41. data/lib/pakyow/data/subscribers/adapters/redis/pipeliner.rb +45 -0
  42. data/lib/pakyow/data/subscribers/adapters/redis/scripts/_shared.lua +73 -0
  43. data/lib/pakyow/data/subscribers/adapters/redis/scripts/expire.lua +16 -0
  44. data/lib/pakyow/data/subscribers/adapters/redis/scripts/persist.lua +15 -0
  45. data/lib/pakyow/data/subscribers/adapters/redis/scripts/register.lua +37 -0
  46. data/lib/pakyow/data/subscribers/adapters/redis.rb +209 -0
  47. data/lib/pakyow/data/subscribers.rb +148 -0
  48. data/lib/pakyow/data/tasks/bootstrap.rake +18 -0
  49. data/lib/pakyow/data/tasks/create.rake +22 -0
  50. data/lib/pakyow/data/tasks/drop.rake +32 -0
  51. data/lib/pakyow/data/tasks/finalize.rake +56 -0
  52. data/lib/pakyow/data/tasks/migrate.rake +24 -0
  53. data/lib/pakyow/data/tasks/reset.rake +18 -0
  54. data/lib/pakyow/data/types.rb +37 -0
  55. data/lib/pakyow/data.rb +27 -0
  56. data/lib/pakyow/environment/data/auto_migrate.rb +31 -0
  57. data/lib/pakyow/environment/data/config.rb +54 -0
  58. data/lib/pakyow/environment/data/connections.rb +76 -0
  59. data/lib/pakyow/environment/data/memory_db.rb +23 -0
  60. data/lib/pakyow/validations/unique.rb +26 -0
  61. metadata +186 -0
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest/sha1"
4
+
5
+ require "concurrent/array"
6
+ require "concurrent/hash"
7
+ require "concurrent/timer_task"
8
+
9
+ require "pakyow/support/deep_dup"
10
+ require "pakyow/support/deep_freeze"
11
+
12
+ module Pakyow
13
+ module Data
14
+ class Subscribers
15
+ module Adapters
16
+ # Manages data subscriptions in memory.
17
+ #
18
+ # Great for development, not for use in production!
19
+ #
20
+ # @api private
21
+ class Memory
22
+ class << self
23
+ def generate_subscription_id(subscription)
24
+ Digest::SHA1.hexdigest(Marshal.dump(subscription))
25
+ end
26
+ end
27
+
28
+ using Support::DeepDup
29
+
30
+ extend Support::DeepFreeze
31
+ unfreezable :subscriptions_by_id, :subscription_ids_by_source, :subscribers_by_subscription_id, :subscription_ids_by_subscriber, :expirations_for_subscriber
32
+
33
+ def initialize(*)
34
+ @subscriptions_by_id = Concurrent::Hash.new
35
+ @subscription_ids_by_source = Concurrent::Hash.new
36
+ @subscribers_by_subscription_id = Concurrent::Hash.new
37
+ @subscription_ids_by_subscriber = Concurrent::Hash.new
38
+ @expirations_for_subscriber = Concurrent::Hash.new
39
+
40
+ Concurrent::TimerTask.new(execution_interval: 10, timeout_interval: 10) {
41
+ @expirations_for_subscriber.each do |subscriber, timeout|
42
+ if timeout < Time.now
43
+ unsubscribe(subscriber)
44
+ end
45
+ end
46
+ }.execute
47
+ end
48
+
49
+ def register_subscriptions(subscriptions, subscriber: nil)
50
+ subscriptions.map { |subscription|
51
+ self.class.generate_subscription_id(subscription).tap do |subscription_id|
52
+ register_subscription_with_subscription_id(subscription, subscription_id)
53
+ register_subscription_id_for_source(subscription_id, subscription[:source])
54
+ register_subscriber_for_subscription_id(subscriber, subscription_id)
55
+ end
56
+ }
57
+ end
58
+
59
+ def subscriptions_for_source(source)
60
+ subscription_ids_for_source(source).map { |subscription_id|
61
+ subscription_with_id(subscription_id)
62
+ }
63
+ end
64
+
65
+ def unsubscribe(subscriber)
66
+ subscription_ids_for_subscriber(subscriber).dup.each do |subscription_id|
67
+ unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
68
+ end
69
+ end
70
+
71
+ def expire(subscriber, seconds)
72
+ @expirations_for_subscriber[subscriber] = Time.now + seconds
73
+ end
74
+
75
+ def persist(subscriber)
76
+ @expirations_for_subscriber.delete(subscriber)
77
+ end
78
+
79
+ def expiring?(subscriber)
80
+ @expirations_for_subscriber.key?(subscriber)
81
+ end
82
+
83
+ def subscribers_for_subscription_id(subscription_id)
84
+ @subscribers_by_subscription_id[subscription_id] || []
85
+ end
86
+
87
+ SERIALIZABLE_IVARS = %i(
88
+ @subscriptions_by_id
89
+ @subscription_ids_by_source
90
+ @subscribers_by_subscription_id
91
+ @expirations_for_subscriber
92
+ ).freeze
93
+
94
+ def serialize
95
+ SERIALIZABLE_IVARS.each_with_object({}) do |ivar, hash|
96
+ hash[ivar] = instance_variable_get(ivar)
97
+ end
98
+ end
99
+
100
+ protected
101
+
102
+ def subscription_ids_for_source(source)
103
+ (@subscription_ids_by_source[source] || []).select { |subscription_id|
104
+ subscribers_for_subscription_id(subscription_id).any? { |subscriber|
105
+ !expiring?(subscriber)
106
+ }
107
+ }
108
+ end
109
+
110
+ def subscription_with_id(subscription_id)
111
+ subscription = @subscriptions_by_id[subscription_id].deep_dup
112
+ subscription[:id] = subscription_id
113
+ subscription
114
+ end
115
+
116
+ def subscription_ids_for_subscriber(subscriber)
117
+ @subscription_ids_by_subscriber[subscriber] || []
118
+ end
119
+
120
+ def register_subscription_with_subscription_id(subscription, subscription_id)
121
+ @subscriptions_by_id[subscription_id] = subscription
122
+ end
123
+
124
+ def register_subscription_id_for_source(subscription_id, source)
125
+ @subscription_ids_by_source[source] ||= Concurrent::Array.new
126
+ (@subscription_ids_by_source[source] << subscription_id).uniq!
127
+ end
128
+
129
+ def register_subscriber_for_subscription_id(subscriber, subscription_id)
130
+ @subscribers_by_subscription_id[subscription_id] ||= Concurrent::Array.new
131
+ (@subscribers_by_subscription_id[subscription_id] << subscriber).uniq!
132
+
133
+ @subscription_ids_by_subscriber[subscriber] ||= Concurrent::Array.new
134
+ (@subscription_ids_by_subscriber[subscriber] << subscription_id).uniq!
135
+ end
136
+
137
+ def unsubscribe_subscriber_from_subscription_id(subscriber, subscription_id)
138
+ subscribers_for_subscription_id(subscription_id).delete(subscriber)
139
+ subscription_ids_for_subscriber(subscriber).delete(subscription_id)
140
+
141
+ if subscribers_for_subscription_id(subscription_id).empty?
142
+ @subscriptions_by_id.delete(subscription_id)
143
+
144
+ @subscription_ids_by_source.each do |_, ids|
145
+ ids.delete(subscription_id)
146
+ end
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pakyow
4
+ module Data
5
+ class Subscribers
6
+ module Adapters
7
+ class Redis
8
+ # @api private
9
+ class Pipeliner
10
+ TIMEOUT = 1.0 / 100.0
11
+
12
+ def initialize(redis)
13
+ @redis = redis
14
+ @commands = []
15
+ end
16
+
17
+ def enqueue(future)
18
+ @commands << { future: future, callback: Proc.new }
19
+ end
20
+
21
+ def wait
22
+ @commands.map { |command|
23
+ while command[:future].value.is_a?(::Redis::FutureNotReady)
24
+ sleep TIMEOUT
25
+ end
26
+
27
+ command[:callback].call(command[:future].value)
28
+ }
29
+ end
30
+
31
+ def self.pipeline(redis)
32
+ pipeliner = Pipeliner.new(redis)
33
+
34
+ redis.pipelined do
35
+ yield pipeliner
36
+ end
37
+
38
+ pipeliner.wait
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,73 @@
1
+ local function massive_redis_command(command, key, t)
2
+ local i = 1
3
+ local temp = {}
4
+
5
+ while(i <= #t) do
6
+ table.insert(temp, t[i + 1])
7
+ table.insert(temp, t[i])
8
+
9
+ if #temp >= 1000 then
10
+ redis.call(command, key, unpack(temp))
11
+ temp = {}
12
+ end
13
+
14
+ i = i + 2
15
+ end
16
+
17
+ if #temp > 0 then
18
+ redis.call(command, key, unpack(temp))
19
+ end
20
+ end
21
+
22
+ local function key_subscription_id(key_prefix, key_delimeter, subscription_id)
23
+ return key_prefix .. key_delimeter .. "subscription:" .. subscription_id
24
+ end
25
+
26
+ local function key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
27
+ return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "subscribers"
28
+ end
29
+
30
+ local function key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id)
31
+ return key_prefix .. key_delimeter .. "subscription:" .. subscription_id .. key_delimeter .. "source"
32
+ end
33
+
34
+ local function key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber)
35
+ return key_prefix .. key_delimeter .. "subscriber:" .. subscriber
36
+ end
37
+
38
+ local function key_subscription_ids_by_source(key_prefix, key_delimeter, source)
39
+ return key_prefix .. key_delimeter .. "source:" .. source
40
+ end
41
+
42
+ local function subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, min, max)
43
+ return redis.call("zrangebyscore", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), min, max)
44
+ end
45
+
46
+ local function expire_subscription(key_prefix, key_delimeter, subscription_id)
47
+ local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
48
+
49
+ -- only expire the subscription if it is not related to a non-expiring subscriber
50
+ if redis.call("zcount", subscription_key, "+inf", "+inf") == 0 then
51
+ local time_expire = redis.call("zrevrangebyscore", subscription_key, "+inf", 0, "WITHSCORES", "LIMIT", 0, 1)[2]
52
+
53
+ if time_expire ~= "inf" then
54
+ local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
55
+ redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), time_expire, subscription_id)
56
+
57
+ redis.call("expireat", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
58
+ redis.call("expireat", subscription_key, time_expire + 1)
59
+ redis.call("expireat", key_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire + 1)
60
+ end
61
+ end
62
+ end
63
+
64
+ local function persist_subscription(key_prefix, key_delimeter, subscription_id)
65
+ local subscription_key = key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id)
66
+
67
+ local source = redis.call("get", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
68
+ redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
69
+
70
+ redis.call("persist", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id))
71
+ redis.call("persist", subscription_key)
72
+ redis.call("persist", key_subscription_id(key_prefix, key_delimeter, subscription_id))
73
+ end
@@ -0,0 +1,16 @@
1
+ local key_prefix = ARGV[1]
2
+ local key_delimeter = ARGV[2]
3
+ local subscriber = ARGV[3]
4
+ local time_expire = ARGV[4]
5
+
6
+ redis.call("expireat", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), time_expire + 1)
7
+
8
+ local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, "+inf", "+inf")
9
+
10
+ local i = 1
11
+ while(i <= #subscription_ids) do
12
+ local subscription_id = subscription_ids[i]
13
+ redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), time_expire, subscriber)
14
+ expire_subscription(key_prefix, key_delimeter, subscription_id)
15
+ i = i + 1
16
+ end
@@ -0,0 +1,15 @@
1
+ local key_prefix = ARGV[1]
2
+ local key_delimeter = ARGV[2]
3
+ local subscriber = ARGV[3]
4
+
5
+ redis.call("persist", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
6
+
7
+ local subscription_ids = subscription_ids_for_subscriber(key_prefix, key_delimeter, subscriber, 0, "+inf")
8
+
9
+ local i = 1
10
+ while(i <= #subscription_ids) do
11
+ local subscription_id = subscription_ids[i]
12
+ redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), "+inf", subscriber)
13
+ persist_subscription(key_prefix, key_delimeter, subscription_id)
14
+ i = i + 1
15
+ end
@@ -0,0 +1,37 @@
1
+ local key_prefix = ARGV[1]
2
+ local key_delimeter = ARGV[2]
3
+ local subscriber = ARGV[3]
4
+ local subscription_id = ARGV[4]
5
+ local subscription_string = ARGV[5]
6
+ local source = ARGV[6]
7
+ local now = ARGV[7]
8
+
9
+ local expiry = "+inf"
10
+
11
+ -- determine if the subscriber is expiring
12
+ local ttl = redis.call("ttl", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber))
13
+
14
+ if ttl > -1 then
15
+ expiry = now + ttl
16
+ end
17
+
18
+ -- store the subscription
19
+ redis.call("set", key_subscription_id(key_prefix, key_delimeter, subscription_id), subscription_string)
20
+
21
+ -- add the subscription to the subscriber's set
22
+ redis.call("zadd", key_subscription_ids_by_subscriber(key_prefix, key_delimeter, subscriber), expiry, subscription_id)
23
+
24
+ -- add the subscriber to the subscription's set
25
+ redis.call("zadd", key_subscribers_by_subscription_id(key_prefix, key_delimeter, subscription_id), expiry, subscriber)
26
+
27
+ -- add the subscription to the source's set
28
+ redis.call("zadd", key_subscription_ids_by_source(key_prefix, key_delimeter, source), "+inf", subscription_id)
29
+
30
+ -- define what source the subscription is for
31
+ redis.call("set", key_source_for_subscription_id(key_prefix, key_delimeter, subscription_id), source)
32
+
33
+ if ttl > -1 then
34
+ expire_subscription(key_prefix, key_delimeter, subscription_id)
35
+ else
36
+ persist_subscription(key_prefix, key_delimeter, subscription_id)
37
+ end
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest/sha1"
4
+
5
+ require "redis"
6
+ require "concurrent/timer_task"
7
+ require "connection_pool"
8
+
9
+ require "pakyow/support/deep_freeze"
10
+
11
+ module Pakyow
12
+ module Data
13
+ class Subscribers
14
+ module Adapters
15
+ # Manages data subscriptions in redis.
16
+ #
17
+ # Use this in production.
18
+ #
19
+ # @api private
20
+ class Redis
21
+ class << self
22
+ def stringify_subscription(subscription)
23
+ Marshal.dump(subscription)
24
+ end
25
+
26
+ def generate_subscription_id(subscription_string)
27
+ Digest::SHA1.hexdigest(subscription_string)
28
+ end
29
+ end
30
+
31
+ SCRIPTS = %i(register expire persist).freeze
32
+ KEY_PART_SEPARATOR = "/"
33
+ KEY_PREFIX = "data"
34
+ INFINITY = "+inf"
35
+
36
+ extend Support::DeepFreeze
37
+ unfreezable :redis
38
+
39
+ def initialize(config)
40
+ @redis = ConnectionPool.new(**config[:pool]) {
41
+ ::Redis.new(config[:connection])
42
+ }
43
+
44
+ @prefix = [config[:key_prefix], KEY_PREFIX].join(KEY_PART_SEPARATOR)
45
+
46
+ @scripts = {}
47
+ load_scripts
48
+
49
+ Concurrent::TimerTask.new(execution_interval: 300, timeout_interval: 300) {
50
+ cleanup
51
+ }.execute
52
+ end
53
+
54
+ def register_subscriptions(subscriptions, subscriber:)
55
+ [].tap do |subscription_ids|
56
+ subscriptions.each do |subscription|
57
+ subscription_string = self.class.stringify_subscription(subscription)
58
+ subscription_id = self.class.generate_subscription_id(subscription_string)
59
+ source = subscription[:source]
60
+
61
+ @redis.with do |redis|
62
+ redis.evalsha(@scripts[:register], argv: [
63
+ @prefix,
64
+ KEY_PART_SEPARATOR,
65
+ subscriber.to_s,
66
+ subscription_id,
67
+ subscription_string,
68
+ source.to_s,
69
+ Time.now.to_i
70
+ ])
71
+ end
72
+
73
+ subscription_ids << subscription_id
74
+ end
75
+ end
76
+ end
77
+
78
+ def subscriptions_for_source(source)
79
+ subscriptions_for_subscription_ids(subscription_ids_for_source(source)).compact
80
+ end
81
+
82
+ def unsubscribe(subscriber)
83
+ expire(subscriber, 0)
84
+ end
85
+
86
+ def expire(subscriber, seconds)
87
+ @redis.with do |redis|
88
+ redis.evalsha(@scripts[:expire], argv: [
89
+ @prefix,
90
+ KEY_PART_SEPARATOR,
91
+ subscriber.to_s,
92
+ Time.now.to_i + seconds
93
+ ])
94
+ end
95
+ end
96
+
97
+ def persist(subscriber)
98
+ @redis.with do |redis|
99
+ redis.evalsha(@scripts[:persist], argv: [
100
+ @prefix,
101
+ KEY_PART_SEPARATOR,
102
+ subscriber.to_s
103
+ ])
104
+ end
105
+ end
106
+
107
+ def expiring?(subscriber)
108
+ @redis.with do |redis|
109
+ redis.ttl(key_subscription_ids_by_subscriber(subscriber)) > -1
110
+ end
111
+ end
112
+
113
+ def subscribers_for_subscription_id(subscription_id)
114
+ @redis.with do |redis|
115
+ redis.zrangebyscore(
116
+ key_subscribers_by_subscription_id(
117
+ subscription_id
118
+ ), INFINITY, INFINITY
119
+ ).map(&:to_sym)
120
+ end
121
+ end
122
+
123
+ def subscription_ids_for_source(source)
124
+ @redis.with do |redis|
125
+ redis.zrangebyscore(
126
+ key_subscription_ids_by_source(
127
+ source
128
+ ), INFINITY, INFINITY
129
+ )
130
+ end
131
+ end
132
+
133
+ # FIXME: Refactor this into a lua script. We'll want to stop using SCAN and instead store
134
+ # known sources in a set. Cleanup should then be based off the set of known sources and
135
+ # return the number of values that were removed.
136
+ #
137
+ def cleanup
138
+ @redis.with do |redis|
139
+ redis.scan_each(match: key_subscription_ids_by_source("*")) do |key|
140
+ Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Cleaning up expired subscriptions for #{key}"
141
+ removed_count = redis.zremrangebyscore(key, 0, Time.now.to_i)
142
+ Pakyow.logger.debug "[Pakyow::Data::Subscribers::Adapters::Redis] Removed #{removed_count} members for #{key}"
143
+ end
144
+ end
145
+ end
146
+
147
+ private
148
+
149
+ def subscriptions_for_subscription_ids(subscription_ids)
150
+ return [] if subscription_ids.empty?
151
+
152
+ @redis.with do |redis|
153
+ redis.mget(subscription_ids.map { |subscription_id|
154
+ key_subscription_id(subscription_id)
155
+ }).zip(subscription_ids).map { |subscription_string, subscription_id|
156
+ begin
157
+ Marshal.restore(subscription_string).tap do |subscription|
158
+ subscription[:id] = subscription_id
159
+ end
160
+ rescue TypeError
161
+ Pakyow.logger.error "could not find subscription for #{subscription_id}"
162
+ {}
163
+ end
164
+ }
165
+ end
166
+ end
167
+
168
+ def build_key(*parts)
169
+ [@prefix].concat(parts).join(KEY_PART_SEPARATOR)
170
+ end
171
+
172
+ def key_subscription_id(subscription_id)
173
+ build_key("subscription:#{subscription_id}")
174
+ end
175
+
176
+ def key_subscribers_by_subscription_id(subscription_id)
177
+ build_key("subscription:#{subscription_id}", "subscribers")
178
+ end
179
+
180
+ def key_subscription_ids_by_subscriber(subscriber)
181
+ build_key("subscriber:#{subscriber}")
182
+ end
183
+
184
+ def key_subscription_ids_by_source(source)
185
+ build_key("source:#{source}")
186
+ end
187
+
188
+ def key_source_for_subscription_id(subscription_id)
189
+ build_key("subscription:#{subscription_id}", "source")
190
+ end
191
+
192
+ def load_scripts
193
+ @redis.with do |redis|
194
+ SCRIPTS.each do |script|
195
+ script_content = File.read(
196
+ File.expand_path("../redis/scripts/_shared.lua", __FILE__)
197
+ ) + File.read(
198
+ File.expand_path("../redis/scripts/#{script}.lua", __FILE__)
199
+ )
200
+
201
+ @scripts[script] = redis.script(:load, script_content)
202
+ end
203
+ end
204
+ end
205
+ end
206
+ end
207
+ end
208
+ end
209
+ end
@@ -0,0 +1,148 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent/executor/thread_pool_executor"
4
+
5
+ require "pakyow/support/core_refinements/method/introspection"
6
+ require "pakyow/support/deep_freeze"
7
+
8
+ module Pakyow
9
+ module Data
10
+ # @api private
11
+ class Subscribers
12
+ attr_accessor :lookup, :adapter
13
+
14
+ extend Support::DeepFreeze
15
+ unfreezable :executor
16
+
17
+ using Support::Refinements::Method::Introspection
18
+
19
+ def initialize(app, adapter = :memory, adapter_config = {})
20
+ @app = app
21
+
22
+ require "pakyow/data/subscribers/adapters/#{adapter}"
23
+ @adapter = Pakyow::Data::Subscribers::Adapters.const_get(
24
+ adapter.to_s.capitalize
25
+ ).new(
26
+ adapter_config.to_h.merge(
27
+ app.config.data.subscriptions.adapter_settings.to_h
28
+ )
29
+ )
30
+
31
+ @executor = Concurrent::ThreadPoolExecutor.new(
32
+ auto_terminate: false,
33
+ min_threads: 1,
34
+ max_threads: 10,
35
+ max_queue: 0
36
+ )
37
+ rescue LoadError, NameError => error
38
+ raise UnknownSubscriberAdapter.build(error, adapter: adapter)
39
+ end
40
+
41
+ def shutdown
42
+ @executor.shutdown
43
+ @executor.wait_for_termination(30)
44
+ end
45
+
46
+ def register_subscriptions(subscriptions, subscriber: nil, &block)
47
+ @executor << Proc.new {
48
+ subscriptions.each do |subscription|
49
+ subscription[:version] = @app.config.data.subscriptions.version
50
+ end
51
+
52
+ @adapter.register_subscriptions(subscriptions, subscriber: subscriber).tap do |ids|
53
+ yield ids if block_given?
54
+ end
55
+ }
56
+ end
57
+
58
+ def did_mutate(source_name, changed_values = nil, result_source = nil)
59
+ @executor << Proc.new {
60
+ begin
61
+ @adapter.subscriptions_for_source(source_name).select { |subscription|
62
+ process?(subscription, changed_values, result_source)
63
+ }.uniq.each do |subscription|
64
+ if subscription[:version] == @app.config.data.subscriptions.version
65
+ process(subscription, result_source)
66
+ end
67
+ end
68
+ rescue => error
69
+ Pakyow.logger.error "[Pakyow::Data::Subscribers] did_mutate failed: #{error}"
70
+ end
71
+ }
72
+ end
73
+
74
+ def unsubscribe(subscriber)
75
+ @adapter.unsubscribe(subscriber)
76
+ end
77
+
78
+ def expire(subscriber, seconds)
79
+ @adapter.expire(subscriber, seconds)
80
+ end
81
+
82
+ def persist(subscriber)
83
+ @adapter.persist(subscriber)
84
+ end
85
+
86
+ private
87
+
88
+ def process?(subscription, changed_values, result_source)
89
+ subscription[:handler] && qualified_subscription?(subscription, changed_values, result_source)
90
+ end
91
+
92
+ def process(subscription, mutated_source)
93
+ callback = subscription[:handler].new(@app)
94
+ arguments = {}
95
+
96
+ if callback.method(:call).keyword_argument?(:id)
97
+ arguments[:id] = subscription[:id]
98
+ end
99
+
100
+ if callback.method(:call).keyword_argument?(:result)
101
+ arguments[:result] = if subscription[:ephemeral]
102
+ mutated_source
103
+ else
104
+ subscription[:proxy]
105
+ end
106
+ end
107
+
108
+ if callback.method(:call).keyword_argument?(:subscription)
109
+ arguments[:subscription] = subscription
110
+ end
111
+
112
+ callback.call(subscription[:payload], **arguments)
113
+ end
114
+
115
+ def qualified_subscription?(subscription, changed_values, result_source)
116
+ if subscription[:ephemeral]
117
+ result_source.qualifications == subscription[:qualifications]
118
+ else
119
+ original_results = if result_source
120
+ result_source.original_results
121
+ else
122
+ []
123
+ end
124
+
125
+ qualified?(
126
+ subscription.delete(:qualifications).to_a,
127
+ changed_values,
128
+ result_source.to_a,
129
+ original_results.to_a
130
+ )
131
+ end
132
+ end
133
+
134
+ QUALIFIABLE_TYPES = [Hash, Support::IndifferentHash]
135
+ def qualified?(qualifications, changed_values, changed_results, original_results)
136
+ qualifications.all? do |key, value|
137
+ (QUALIFIABLE_TYPES.include?(changed_values.class) && changed_values.to_h[key] == value) || qualified_result?(key, value, changed_results, original_results)
138
+ end
139
+ end
140
+
141
+ def qualified_result?(key, value, changed_results, original_results)
142
+ original_results.concat(changed_results).any? do |result|
143
+ result[key] == value
144
+ end
145
+ end
146
+ end
147
+ end
148
+ end